validmind 2.1.1__py3-none-any.whl → 2.2.4__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionModelInsampleComparison.py

@@ -106,7 +106,7 @@ class RegressionModelInsampleComparison(Metric):
         evaluation_results = []
 
         for i, model in enumerate(models):
-            X_columns = dataset.get_features_columns()
+            X_columns = dataset.feature_columns
             y_true = dataset.y
             y_pred = dataset.y_pred(model)
 

validmind/tests/model_validation/statsmodels/RegressionModelOutsampleComparison.py

@@ -96,7 +96,7 @@ class RegressionModelOutsampleComparison(Metric):
 
         for fitted_model in model_list:
             # Extract the column names of the independent variables from the model
-            independent_vars = dataset.get_features_columns()
+            independent_vars = dataset.feature_columns
 
             # Separate the target variable and features in the test dataset
             y_test = dataset.y

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -57,7 +57,7 @@ class RegressionModelSummary(Metric):
     }
 
     def run(self):
-        X_columns = self.inputs.dataset.get_features_columns()
+        X_columns = self.inputs.dataset.feature_columns
 
         y_true = self.inputs.dataset.y
         y_pred = self.inputs.dataset.y_pred(self.inputs.model)

validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py

@@ -73,7 +73,7 @@ class RegressionModelsCoeffs(Metric):
             raise ValueError("List of models must be provided in the models parameter")
 
         for model in self.inputs.models:
-            if model.model_class() != "statsmodels" and model.model_class() != "R":
+            if model.class_ != "statsmodels" and model.class_ != "R":
                 raise SkipTestError(
                     "Only statsmodels and R models are supported for this metric"
                 )

validmind/tests/model_validation/statsmodels/RegressionModelsPerformance.py

@@ -80,7 +80,7 @@ class RegressionModelsPerformance(Metric):
         evaluation_results = []
 
         for model, dataset in zip(models, datasets):
-            X_columns = dataset.get_features_columns()
+            X_columns = dataset.feature_columns
             y_true = dataset.y
             y_pred = dataset.y_pred(model)
 

validmind/tests/model_validation/statsmodels/ScorecardHistogram.py

@@ -112,16 +112,15 @@ class ScorecardHistogram(Metric):
         dataframes = []
         metric_value = {"score_histogram": {}}
         for dataset in self.inputs.datasets:
-            df = dataset.df.copy()
-            # Check if the score_column exists in the DataFrame
-            if score_column not in df.columns:
+            if score_column not in dataset.df.columns:
                 raise ValueError(
                     f"The required column '{score_column}' is not present in the dataset with input_id {dataset.input_id}"
                 )
 
-            df[score_column] = dataset.get_extra_column(score_column)
-            dataframes.append(df)
-            metric_value["score_histogram"][dataset.input_id] = list(df[score_column])
+            dataframes.append(dataset.df.copy())
+            metric_value["score_histogram"][dataset.input_id] = list(
+                dataset.df[score_column]
+            )
 
         figures = self.plot_score_histogram(
             dataframes, dataset_titles, score_column, target_column, title

validmind/unit_metrics/__init__.py

@@ -6,8 +6,6 @@ import hashlib
 import json
 from importlib import import_module
 
-import numpy as np
-
 from ..tests.decorator import _build_result, _inspect_signature
 from ..utils import get_model_info, test_id_to_name
 
@@ -58,7 +56,7 @@ def _serialize_model(model):
     return hash_object.hexdigest()
 
 
-def _serialize_dataset(dataset, model_id):
+def _serialize_dataset(dataset, model):
     """
     Serialize the description of the dataset input to a unique hash.
 
@@ -68,11 +66,11 @@ def _serialize_dataset(dataset, model_id):
 
     Args:
         dataset: The dataset object, which should have properties like _df (pandas DataFrame),
-                 target_column (string), feature_columns (list of strings), and _extra_columns (dict).
-        model_id (str): The ID of the model associated with the prediction column.
+                 target_column (string), feature_columns (list of strings), and extra_columns (dict).
+        model (VMModel): The model whose predictions will be included in the serialized dataset
 
     Returns:
-        str: A SHA-256 hash representing the dataset.
+        str: MD5 hash of the dataset
 
     Note:
         Including the model ID and prediction column name in the hash calculation ensures uniqueness,
@@ -80,57 +78,33 @@ def _serialize_dataset(dataset, model_id):
         This approach guarantees that the hash will distinguish between model-generated predictions
         and pre-computed prediction columns, addressing potential hash collisions.
     """
-
-    # Access the prediction column for the given model ID from the dataset's extra columns
-    prediction_column_name = dataset._extra_columns["prediction_columns"][model_id]
-
-    # Include model ID and prediction column name directly in the hash calculation
-    model_and_prediction_info = f"{model_id}_{prediction_column_name}".encode()
-
-    # Start with target and feature columns, and include the prediction column
-    columns = (
-        [dataset._target_column] + dataset._feature_columns + [prediction_column_name]
+    return _fast_hash(
+        dataset.df[
+            [
+                *dataset.feature_columns,
+                dataset.target_column,
+                dataset.prediction_column(model),
+            ]
+        ]
     )
 
-    # Use _fast_hash function and include model_and_prediction_info in the hash calculation
-    hash_digest = _fast_hash(
-        dataset._df[columns], model_and_prediction_info=model_and_prediction_info
-    )
-
-    return hash_digest
-
 
-def _fast_hash(df, sample_size=1000, model_and_prediction_info=None):
+def _fast_hash(df, sample_size=1000):
     """
-    Generates a hash for a DataFrame by sampling and combining its size, content,
-    and optionally model and prediction information.
+    Generates a fast hash by sampling, converting to string and md5 hashing.
 
     Args:
         df (pd.DataFrame): The DataFrame to hash.
         sample_size (int): The maximum number of rows to include in the sample.
-        model_and_prediction_info (bytes, optional): Additional information to include in the hash.
 
     Returns:
-        str: A SHA-256 hash of the DataFrame's sample and additional information.
+        str: MD5 hash of the DataFrame.
     """
-    # Convert the number of rows to bytes and include it in the hash calculation
-    rows_bytes = str(len(df)).encode()
+    df_sample = df.sample(n=min(sample_size, len(df)), random_state=42)
 
-    # Sample rows if DataFrame is larger than sample_size, ensuring reproducibility
-    if len(df) > sample_size:
-        df_sample = df.sample(n=sample_size, random_state=42)
-    else:
-        df_sample = df
-
-    # Convert the sampled DataFrame to a byte array. np.asarray ensures compatibility with various DataFrame contents.
-    byte_array = np.asarray(df_sample).data.tobytes()
-
-    # Initialize the hash object and update it with the row count, data bytes, and additional info
-    hash_obj = hashlib.sha256(
-        rows_bytes + byte_array + (model_and_prediction_info or b"")
-    )
-
-    return hash_obj.hexdigest()
+    return hashlib.md5(
+        df_sample.to_string(header=True, index=True).encode()
+    ).hexdigest()
 
 
 def get_metric_cache_key(metric_id, params, inputs):
@@ -150,9 +124,8 @@ def get_metric_cache_key(metric_id, params, inputs):
 
     dataset = inputs["dataset"]
     model = inputs["model"]
-    model_id = model.input_id
 
-    cache_elements.append(_serialize_dataset(dataset, model_id))
+    cache_elements.append(_serialize_dataset(dataset, model))
 
     cache_elements.append(_serialize_model(model))
 
@@ -197,7 +170,11 @@ def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False)
             **{k: v for k, v in inputs.items() if k in _inputs.keys()},
             **{k: v for k, v in params.items() if k in _params.keys()},
         )
-        unit_metric_results_cache[cache_key] = (result, list(_inputs.keys()))
+        unit_metric_results_cache[cache_key] = (
+            result,
+            # store the input ids that were used to calculate the result
+            [v.input_id for v in inputs.values()],
+        )
 
     value = unit_metric_results_cache[cache_key][0]
 
@@ -235,7 +212,7 @@ def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False)
     )
 
     # in case the user tries to log the result object
-    def log(self):
+    def log():
         raise Exception(
             "Cannot log unit metrics directly..."
             "You can run this unit metric as part of a composite metric and log that"

validmind/unit_metrics/composite.py

@@ -8,7 +8,7 @@ from uuid import uuid4
 
 from ..logging import get_logger
 from ..tests.decorator import _inspect_signature
-from ..utils import run_async, test_id_to_name
+from ..utils import get_description_metadata, run_async, test_id_to_name
 from ..vm_models.test.metric import Metric
 from ..vm_models.test.metric_result import MetricResult
 from ..vm_models.test.result_summary import ResultSummary, ResultTable
@@ -37,6 +37,7 @@ class CompositeMetric(Metric):
             metric_ids=self.unit_metrics,
             description=self.description(),
             inputs=self._get_input_dict(),
+            accessed_inputs=self.get_accessed_inputs(),
             params=self.params,
             output_template=self.output_template,
             show=False,
@@ -103,6 +104,7 @@ def run_metrics(
     description: str = None,
     output_template: str = None,
     inputs: dict = None,
+    accessed_inputs: List[str] = None,
     params: dict = None,
     test_id: str = None,
     show: bool = True,
@@ -128,6 +130,8 @@ def run_metrics(
         output_template (_type_, optional): Output template to customize the result
             table.
         inputs (_type_, optional): Inputs to pass to the unit metrics. Defaults to None
+        accessed_inputs (_type_, optional): Inputs that were accessed when running the
+            unit metrics - used for input tracking. Defaults to None.
         params (_type_, optional): Parameters to pass to the unit metrics. Defaults to
             None.
         test_id (str, optional): Test ID of the composite metric. Required if name is
@@ -196,13 +200,15 @@ def run_metrics(
         </style>
         """
 
+    result_summary = ResultSummary(results=[ResultTable(data=[results])])
     result_wrapper = MetricResultWrapper(
         result_id=test_id,
         result_metadata=[
-            {
-                "content_id": f"metric_description:{test_id}",
-                "text": description,
-            },
+            get_description_metadata(
+                test_id=test_id,
+                default_description=description,
+                summary=result_summary.serialize(),
+            ),
             {
                 "content_id": f"composite_metric_def:{test_id}:unit_metrics",
                 "json": metric_ids,
@@ -212,13 +218,13 @@ def run_metrics(
                 "json": {"output_template": output_template},
             },
         ],
-        inputs=list(inputs.keys()),
+        inputs=accessed_inputs,
         output_template=output_template,
         metric=MetricResult(
             key=test_id,
             ref_id=str(uuid4()),
             value=results,
-            summary=ResultSummary(results=[ResultTable(data=[results])]),
+            summary=result_summary,
         ),
     )
 

validmind/unit_metrics/regression/sklearn/AdjustedRSquaredScore.py

@@ -16,6 +16,6 @@ def AdjustedRSquaredScore(model, dataset):
     )
 
     row_count = len(dataset.y)
-    feature_count = len(dataset.get_features_columns())
+    feature_count = len(dataset.feature_columns)
 
     return 1 - (1 - r2_score) * (row_count - 1) / (row_count - feature_count)

validmind/utils.py

@@ -6,22 +6,29 @@ import asyncio
 import difflib
 import json
 import math
+import os
 import re
 import sys
 from platform import python_version
 from typing import Any
 
 import matplotlib.pylab as pylab
+import mistune
 import nest_asyncio
 import numpy as np
 import pandas as pd
 import seaborn as sns
 from IPython.core import getipython
-from IPython.display import HTML, display
+from IPython.display import HTML
+from IPython.display import display as ipy_display
+from latex2mathml.converter import convert
 from matplotlib.axes._axes import _log as matplotlib_axes_logger
 from numpy import ndarray
 from tabulate import tabulate
 
+from .ai import generate_description
+from .html_templates.content_blocks import math_jax_snippet, python_syntax_highlighting
+
 DEFAULT_BIG_NUMBER_DECIMALS = 2
 DEFAULT_SMALL_NUMBER_DECIMALS = 4
 
@@ -97,6 +104,8 @@ class NumpyEncoder(json.JSONEncoder):
             return bool(obj)
         if isinstance(obj, pd.Timestamp):
             return str(obj)
+        if isinstance(obj, set):
+            return list(obj)
         return super().default(obj)
 
     def encode(self, obj):
@@ -345,10 +354,10 @@ def test_id_to_name(test_id: str) -> str:
 
 def get_model_info(model):
     """Attempts to extract all model info from a model object instance"""
-    architecture = model.model_name()
-    framework = model.model_library()
-    framework_version = model.model_library_version()
-    language = model.model_language()
+    architecture = model.name
+    framework = model.library
+    framework_version = model.library_version
+    language = model.language
 
     if language is None:
         language = f"Python {python_version()}"
@@ -402,4 +411,88 @@ def preview_test_config(config):
     <div id="collapsibleContent" style="display:none;"><pre>{formatted_json}</pre></div>
     """
 
-    display(HTML(collapsible_html))
+    ipy_display(HTML(collapsible_html))
+
+
+def display(widget_or_html, syntax_highlighting=True, mathjax=True):
+    """Display widgets with extra goodies (syntax highlighting, MathJax, etc.)"""
+    if isinstance(widget_or_html, str):
+        ipy_display(HTML(widget_or_html))
+        # if html we can auto-detect if we actually need syntax highlighting or MathJax
+        syntax_highlighting = 'class="language-' in widget_or_html
+        mathjax = "$$" in widget_or_html
+    else:
+        ipy_display(widget_or_html)
+
+    if syntax_highlighting:
+        ipy_display(HTML(python_syntax_highlighting))
+
+    if mathjax:
+        ipy_display(HTML(math_jax_snippet))
+
+
+def md_to_html(md: str, mathml=False) -> str:
+    """Converts Markdown to HTML using mistune with plugins"""
+    # use mistune with math plugin to convert to html
+    html = mistune.create_markdown(
+        plugins=["math", "table", "strikethrough", "footnotes"]
+    )(md)
+
+    if not mathml:
+        # return the html as is (with latex that will be rendered by MathJax)
+        return html
+
+    # convert the latex to MathML which CKeditor can render
+    math_block_pattern = re.compile(r'<div class="math">\$\$([\s\S]*?)\$\$</div>')
+    html = math_block_pattern.sub(
+        lambda match: "<p>{}</p>".format(convert(match.group(1), display="block")), html
+    )
+
+    inline_math_pattern = re.compile(r'<span class="math">\\\((.*?)\\\)</span>')
+    html = inline_math_pattern.sub(
+        lambda match: "<span>{}</span>".format(
+            convert(match.group(1), display="inline")
+        ),
+        html,
+    )
+
+    return html
+
+
+def get_description_metadata(test_id, default_description, summary=None, figures=None):
+    """Get Metadata Dictionary for a Test or Metric Result
+
+    Generates an LLM interpretation of the test results or uses the default
+    description and returns a metadata object that can be logged with the test results.
+
+    To enable LLM-generated descriptions, set the VALIDMIND_LLM_DESCRIPTIONS_ENABLED
+    environment variable to "true". The default description will be used if LLM
+    descriptions are disabled.
+
+    Note: Either the summary or figures must be provided to generate the description.
+
+    Args:
+        test_id (str): The test ID
+        default_description (str): The default description for the test
+        summary (Any): The test summary or results to interpret
+        figures (List[Figure]): The figures to attach to the test suite result
+
+    Returns:
+        dict: The metadata object to be logged with the test results
+    """
+    if os.environ.get("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "false").lower() == "true":
+        revision_name = "Generated by ValidMind AI"
+        description = generate_description(
+            test_name=test_id,
+            test_description=default_description,
+            test_summary=summary,
+            figures=figures,
+        )
+    else:
+        revision_name = "Default Description"
+        description = default_description
+
+    return {
+        "content_id": f"metric_description:{test_id}::{revision_name}",
+        "text": description,
+    }

validmind/vm_models/__init__.py

@@ -6,7 +6,7 @@
 Models entrypoint
 """
 
-from .dataset import VMDataset
+from .dataset.dataset import VMDataset
 from .figure import Figure
 from .model import R_MODEL_TYPES, ModelAttributes, VMModel
 from .test.metric import Metric

validmind/vm_models/dataset/__init__.py

@@ -0,0 +1,7 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from .dataset import DataFrameDataset, PolarsDataset, TorchDataset, VMDataset
+
+__all__ = ["VMDataset", "DataFrameDataset", "PolarsDataset", "TorchDataset"]