validmind 2.5.6__py3-none-any.whl → 2.5.15__py3-none-any.whl

validmind/unit_metrics/__init__.py

@@ -2,145 +2,111 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import glob
 import hashlib
 import json
+import os
 from importlib import import_module
+from textwrap import dedent
+
+from IPython.display import Markdown, display
 
 from validmind.input_registry import input_registry
 from validmind.tests.decorator import _build_result, _inspect_signature
-from validmind.utils import get_model_info, test_id_to_name
+from validmind.utils import test_id_to_name
 
 unit_metric_results_cache = {}
 
 
-def _serialize_params(params):
-    """
-    Serialize the parameters to a unique hash, handling None values.
-    This function serializes the parameters dictionary to a JSON string,
-    then creates a SHA-256 hash of the string to ensure a unique identifier
-    for the parameters. If params is None, a default hash is returned.
-
-    Args:
-        params (dict or None): The parameters to be serialized.
-
-    Returns:
-        str: A SHA-256 hash of the JSON string representation of the params,
-             or a default hash if params is None.
-    """
-    if params is None:
-        # Handle None by returning a hash of an empty dictionary or a predefined value
-        params_json = json.dumps({})
-    else:
-        params_json = json.dumps(params, sort_keys=True)
-
-    hash_object = hashlib.sha256(params_json.encode())
-    return hash_object.hexdigest()
-
-
-def _serialize_model(model):
-    """
-    Generate a SHA-256 hash for a scikit-learn model based on its type and parameters.
-
-    Args:
-        model VMModel: The model to be serialized.
-
-    Returns:
-        str: A SHA-256 hash of the model's description.
-    """
-
-    model_info = get_model_info(model)
+def _serialize_dataset(dataset, model=None, sample_size=1000):
+    columns = [*dataset.feature_columns, dataset.target_column]
+    if model:
+        columns.append(dataset.prediction_column(model))
 
-    model_json = json.dumps(model_info, sort_keys=True)
+    df = dataset._df[columns]
 
-    # Create a SHA-256 hash of the JSON string
-    hash_object = hashlib.sha256(model_json.encode())
-    return hash_object.hexdigest()
+    return hashlib.md5(
+        df.sample(n=min(sample_size, df.shape[0]), random_state=42)
+        .to_string(header=True, index=True)
+        .encode()
+    ).hexdigest()
 
 
-def _serialize_dataset(dataset, model):
-    """
-    Serialize the description of the dataset input to a unique hash.
+def _get_metric_cache_key(metric_id, inputs, params):
+    cache_elements = [
+        metric_id,
+        hashlib.md5(json.dumps(params, sort_keys=True).encode()).hexdigest(),
+    ]
 
-    This function generates a hash based on the dataset's structure, including
-    the target and feature columns, the prediction column associated with a specific model ID,
-    and directly incorporates the model ID and prediction column name to ensure uniqueness.
+    if "model" in inputs:
+        cache_elements.append(inputs["model"].input_id)
 
-    Args:
-        dataset: The dataset object, which should have properties like _df (pandas DataFrame),
-                 target_column (string), feature_columns (list of strings), and extra_columns (dict).
-        model (VMModel): The model whose predictions will be included in the serialized dataset
-
-    Returns:
-        str: MD5 hash of the dataset
+    if "dataset" in inputs:
+        cache_elements.append(inputs["dataset"].input_id)
+        cache_elements.append(
+            _serialize_dataset(inputs["dataset"], inputs.get("model"))
+        )
 
-    Note:
-        Including the model ID and prediction column name in the hash calculation ensures uniqueness,
-        especially in cases where the predictions are sparse or the dataset has not significantly changed.
-        This approach guarantees that the hash will distinguish between model-generated predictions
-        and pre-computed prediction columns, addressing potential hash collisions.
-    """
-    return _fast_hash(
-        dataset._df[
-            [
-                *dataset.feature_columns,
-                dataset.target_column,
-                dataset.prediction_column(model),
-            ]
-        ]
-    )
+    return hashlib.md5("_".join(cache_elements).encode()).hexdigest()
 
 
-def _fast_hash(df, sample_size=1000):
-    """
-    Generates a fast hash by sampling, converting to string and md5 hashing.
+def describe_metric(metric_id, raw=False):
+    """Describe a metric
 
     Args:
-        df (pd.DataFrame): The DataFrame to hash.
-        sample_size (int): The maximum number of rows to include in the sample.
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.F1')
+        raw (bool): Whether to return the description as a dictionary
 
     Returns:
-        str: MD5 hash of the DataFrame.
+        dict: A dictionary containing the metric description
     """
-    df_sample = df.sample(n=min(sample_size, len(df)), random_state=42)
-
-    return hashlib.md5(
-        df_sample.to_string(header=True, index=True).encode()
-    ).hexdigest()
+    metric = load_metric(metric_id)
+    inputs, params = _inspect_signature(metric)
 
+    if raw:
+        return {
+            "id": metric_id,
+            "description": metric.__doc__,
+            "inputs": inputs,
+            "params": params,
+        }
 
-def get_metric_cache_key(metric_id, params, inputs):
-    cache_elements = [metric_id]
+    inputs = ", ".join(inputs.keys())
+    params = ", ".join(params.keys())
+    description_md = f"""
+    ### {test_id_to_name(metric_id)} (*'{metric_id}'*)
 
-    # Serialize params if not None
-    serialized_params = _serialize_params(params) if params else "None"
-    cache_elements.append(serialized_params)
+    {metric.__doc__ or ""}
 
-    # Check if 'inputs' is a dictionary
-    if not isinstance(inputs, dict):
-        raise TypeError("Expected 'inputs' to be a dictionary.")
+    **Inputs**: {inputs}
 
-    # Check for 'model' and 'dataset' keys in 'inputs'
-    if "model" not in inputs or "dataset" not in inputs:
-        raise ValueError("Missing 'model' or 'dataset' in 'inputs'.")
-
-    dataset = inputs["dataset"]
-    model = inputs["model"]
+    **Parameters**: {params}
+    """
+    display(Markdown(dedent(description_md)))
 
-    cache_elements.append(_serialize_dataset(dataset, model))
 
-    cache_elements.append(_serialize_model(model))
+def list_metrics():
+    """List all available metrics
 
-    # Combine elements to form the cache key
-    combined_elements = "_".join(cache_elements)
-    key = hashlib.sha256(combined_elements.encode()).hexdigest()
-    return key
+    Returns:
+        list: A list of metric ids
+    """
+    # current directory of this file is the __init__.py file in the validmind/unit_metrics directory
+    # glob for all metrics in the unit_metrics directory (indicated by capitalized python files)
+    # recursive since we want to include subdirectories
+    curr_dir = os.path.dirname(os.path.realpath(__file__))
+    return [
+        f"{__name__}.{os.path.relpath(metric, curr_dir).replace('/', '.')[:-3]}"
+        for metric in glob.glob(f"{curr_dir}/**/*.py", recursive=True)
+        if os.path.isfile(metric) and os.path.basename(metric)[0].isupper()
+    ]
 
 
 def load_metric(metric_id):
     """Load a metric class from a string
 
     Args:
-        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.sklearn.F1')
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.F1')
 
     Returns:
         callable: The metric function
@@ -152,7 +118,7 @@ def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False)
     """Run a single metric and cache the results
 
     Args:
-        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.sklearn.F1')
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.F1')
         inputs (dict): A dictionary of the metric inputs
         params (dict): A dictionary of the metric parameters
         show (bool): Whether to display the results
@@ -164,7 +130,7 @@ def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False)
     }
     params = params or {}
 
-    cache_key = get_metric_cache_key(metric_id, params, inputs)
+    cache_key = _get_metric_cache_key(metric_id, inputs, params)
 
     if cache_key not in unit_metric_results_cache:
         metric = load_metric(metric_id)
@@ -182,53 +148,24 @@ def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False)
             result,
             # store the input ids that were used to calculate the result
             [v.input_id for v in inputs.values()],
+            # store the params that were used to calculate the result
+            params,
         )
 
-    value = unit_metric_results_cache[cache_key][0]
+    cached_result = unit_metric_results_cache[cache_key]
 
     if value_only:
-        return value
-
-    output_template = f"""
-    <table>
-        <thead>
-            <tr>
-                <th>Metric</th>
-                <th>Value</th>
-            </tr>
-        </thead>
-        <tbody>
-            <tr>
-                <td><strong>{test_id_to_name(metric_id)}</strong></td>
-                <td>{value:.4f}</td>
-            </tr>
-        </tbody>
-    </table>
-    <style>
-        th, td {{
-            padding: 5px;
-            text-align: left;
-        }}
-    </style>
-    """
-    result = _build_result(
-        results=value,
+        return cached_result[0]
+
+    result_wrapper = _build_result(
+        results=cached_result[0],
         test_id=metric_id,
-        description="",
-        output_template=output_template,
-        inputs=unit_metric_results_cache[cache_key][1],
+        inputs=cached_result[1],
+        params=cached_result[2],
+        generate_description=False,
     )
 
-    # in case the user tries to log the result object
-    def log():
-        raise Exception(
-            "Cannot log unit metrics directly..."
-            "You can run this unit metric as part of a composite metric and log that"
-        )
-
-    result.log = log
-
     if show:
-        result.show()
+        result_wrapper.show()
 
-    return result
+    return result_wrapper

validmind/unit_metrics/classification/{sklearn/Accuracy.py → Accuracy.py}

@@ -7,8 +7,8 @@ from sklearn.metrics import accuracy_score
 from validmind import tags, tasks
 
 
-@tags("classification", "sklearn", "unit_metric")
 @tasks("classification")
+@tags("classification")
 def Accuracy(dataset, model):
     """Calculates the accuracy of a model"""
     return accuracy_score(dataset.y, dataset.y_pred(model))

validmind/unit_metrics/classification/{sklearn/F1.py → F1.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import f1_score
 from validmind import tags, tasks
 
 
-@tags("classification", "sklearn", "unit_metric")
 @tasks("classification")
+@tags("classification")
 def F1(model, dataset, **kwargs):
     return f1_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/{sklearn/Precision.py → Precision.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import precision_score
 from validmind import tags, tasks
 
 
-@tags("classification", "sklearn", "unit_metric")
 @tasks("classification")
+@tags("classification")
 def Precision(model, dataset, **kwargs):
     return precision_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/{sklearn/ROC_AUC.py → ROC_AUC.py}

@@ -9,10 +9,9 @@ from sklearn.preprocessing import LabelBinarizer
 from validmind import tags, tasks
 
 
-@tags("classification", "sklearn", "unit_metric")
 @tasks("classification")
+@tags("classification")
 def ROC_AUC(model, dataset, **kwargs):
-
     y_true = dataset.y
 
     if len(unique(y_true)) > 2:

validmind/unit_metrics/classification/{sklearn/Recall.py → Recall.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import recall_score
 from validmind import tags, tasks
 
 
-@tags("classification", "sklearn", "unit_metric")
 @tasks("classification")
+@tags("classification")
 def Recall(model, dataset, **kwargs):
     return recall_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/{sklearn/AdjustedRSquaredScore.py → AdjustedRSquaredScore.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import r2_score as _r2_score
 from validmind import tags, tasks
 
 
-@tags("regression", "sklearn", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def AdjustedRSquaredScore(model, dataset):
     r2_score = _r2_score(

validmind/unit_metrics/regression/GiniCoefficient.py

@@ -7,7 +7,7 @@ import numpy as np
 from validmind import tags, tasks
 
 
-@tags("regression", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def GiniCoefficient(dataset, model):
     y_true = dataset.y

validmind/unit_metrics/regression/HuberLoss.py

@@ -7,7 +7,7 @@ import numpy as np
 from validmind import tags, tasks
 
 
-@tags("regression", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def HuberLoss(model, dataset):
     y_true = dataset.y

validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py

@@ -7,7 +7,7 @@ import numpy as np
 from validmind import tags, tasks
 
 
-@tags("regression", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def KolmogorovSmirnovStatistic(dataset, model):
     y_true = dataset.y.flatten()

validmind/unit_metrics/regression/{sklearn/MeanAbsoluteError.py → MeanAbsoluteError.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import mean_absolute_error as _mean_absolute_error
 from validmind import tags, tasks
 
 
-@tags("regression", "sklearn", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def MeanAbsoluteError(model, dataset, **kwargs):
     return _mean_absolute_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/MeanAbsolutePercentageError.py

@@ -7,7 +7,7 @@ import numpy as np
 from validmind import tags, tasks
 
 
-@tags("regression", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def MeanAbsolutePercentageError(model, dataset):
     y_true = dataset.y

validmind/unit_metrics/regression/MeanBiasDeviation.py

@@ -7,7 +7,7 @@ import numpy as np
 from validmind import tags, tasks
 
 
-@tags("regression", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def MeanBiasDeviation(model, dataset):
     return np.mean(dataset.y - dataset.y_pred(model))

validmind/unit_metrics/regression/{sklearn/MeanSquaredError.py → MeanSquaredError.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import mean_squared_error
 from validmind import tags, tasks
 
 
-@tags("regression", "sklearn", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def MeanSquaredError(model, dataset, **kwargs):
     return mean_squared_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/QuantileLoss.py

@@ -7,7 +7,7 @@ import numpy as np
 from validmind import tags, tasks
 
 
-@tags("regression", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def QuantileLoss(model, dataset, quantile=0.5):
     error = dataset.y - dataset.y_pred(model)

validmind/unit_metrics/regression/{sklearn/RSquaredScore.py → RSquaredScore.py}

@@ -7,7 +7,7 @@ from sklearn.metrics import r2_score
 from validmind import tags, tasks
 
 
-@tags("regression", "sklearn", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def RSquaredError(model, dataset):
     return r2_score(dataset.y, dataset.y_pred(model))

validmind/unit_metrics/regression/{sklearn/RootMeanSquaredError.py → RootMeanSquaredError.py}

@@ -8,7 +8,7 @@ from sklearn.metrics import mean_squared_error
 from validmind import tags, tasks
 
 
-@tags("regression", "sklearn", "unit_metric")
+@tags("regression")
 @tasks("regression")
 def RootMeanSquaredError(model, dataset, **kwargs):
     return np.sqrt(

validmind/vm_models/dataset/dataset.py

@@ -323,6 +323,7 @@ class VMDataset(VMInput):
 
         if column_name and column_name in self.feature_columns:
             self.feature_columns.remove(column_name)
+            self._set_feature_columns(self.feature_columns)
 
         return self.extra_columns.prediction_column(model, column_name)
 
@@ -333,6 +334,7 @@ class VMDataset(VMInput):
 
         if column_name and column_name in self.feature_columns:
             self.feature_columns.remove(column_name)
+            self._set_feature_columns(self.feature_columns)
 
         return self.extra_columns.probability_column(model, column_name)
 

validmind/vm_models/figure.py

@@ -157,6 +157,11 @@ class Figure:
 
             return f"data:image/png;base64,{b64_data}"
 
+        elif is_png_image(self.figure):
+            b64_data = base64.b64encode(self.figure).decode("utf-8")
+
+            return f"data:image/png;base64,{b64_data}"
+
         raise UnsupportedFigureError(
             f"Unrecognized figure type: {get_full_typename(self.figure)}"
         )