validmind 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/unit_metrics/__init__.py

@@ -3,14 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import hashlib
-import importlib
 import json
+from importlib import import_module
 
 import numpy as np
 
-from validmind.vm_models import TestInput
-
-from ..utils import get_model_info
+from ..tests.decorator import _build_result, _inspect_signature
+from ..utils import get_model_info, test_id_to_name
 
 unit_metric_results_cache = {}
 
@@ -134,52 +133,6 @@ def _fast_hash(df, sample_size=1000, model_and_prediction_info=None):
     return hash_obj.hexdigest()
 
 
-def _get_metric_class(metric_id):
-    """Get the metric class by metric_id
-
-    This function will load the metric class by metric_id.
-
-    Args:
-        metric_id (str): The full metric id (e.g. 'validmind.vm_models.test.v2.model_validation.sklearn.F1')
-
-    Returns:
-        Metric: The metric class
-    """
-
-    metric_module = importlib.import_module(f"{metric_id}")
-
-    class_name = metric_id.split(".")[-1]
-
-    # Access the class within the F1 module
-    metric_class = getattr(metric_module, class_name)
-
-    return metric_class
-
-
-def get_input_type(input_obj):
-    """
-    Determines whether the input object is a 'dataset' or 'model' based on its class module path.
-
-    Args:
-        input_obj: The object to type check.
-
-    Returns:
-        str: 'dataset' or 'model' depending on the object's module, or raises ValueError.
-    """
-    # Obtain the class object of input_obj (for clarity and debugging)
-    class_obj = input_obj.__class__
-
-    # Obtain the module name as a string from the class object
-    class_module = class_obj.__module__
-
-    if "validmind.vm_models.dataset" in class_module:
-        return "dataset"
-    elif "validmind.models" in class_module:
-        return "model"
-    else:
-        raise ValueError("Input must be of type validmind Dataset or Model")
-
-
 def get_metric_cache_key(metric_id, params, inputs):
     cache_elements = [metric_id]
 
@@ -209,36 +162,88 @@ def get_metric_cache_key(metric_id, params, inputs):
     return key
 
 
-def run_metric(metric_id=None, inputs=None, params=None):
-    """Run a single metric
-
-    This function provides a high level interface for running a single metric. A metric
-    is a single test that calculates a value based on the input data.
+def load_metric(metric_id):
+    """Load a metric class from a string
 
     Args:
-        metric_id (str): The metric name (e.g. 'F1')
-        params (dict): A dictionary of the metric parameters
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.sklearn.F1')
 
     Returns:
-        MetricResult: The metric result object
+        callable: The metric function
     """
-    cache_key = get_metric_cache_key(metric_id, params, inputs)
+    return getattr(import_module(metric_id), metric_id.split(".")[-1])
 
-    # Check if the metric value already exists in the global variable
-    if cache_key in unit_metric_results_cache:
-        return unit_metric_results_cache[cache_key]
 
-    # Load the metric class by metric_id
-    metric_class = _get_metric_class(metric_id)
+def run_metric(metric_id, inputs=None, params=None, show=True, value_only=False):
+    """Run a single metric and cache the results
 
-    # Initialize the metric
-    metric = metric_class(test_id=metric_id, inputs=TestInput(inputs), params=params)
-
-    # Run the metric
-    result = metric.run()
+    Args:
+        metric_id (str): The metric id (e.g. 'validmind.unit_metrics.classification.sklearn.F1')
+        inputs (dict): A dictionary of the metric inputs
+        params (dict): A dictionary of the metric parameters
+        show (bool): Whether to display the results
+        value_only (bool): Whether to return only the value
+    """
+    inputs = inputs or {}
+    params = params or {}
 
     cache_key = get_metric_cache_key(metric_id, params, inputs)
 
-    unit_metric_results_cache[cache_key] = result
+    if cache_key not in unit_metric_results_cache:
+        metric = load_metric(metric_id)
+        _inputs, _params = _inspect_signature(metric)
+
+        result = metric(
+            **{k: v for k, v in inputs.items() if k in _inputs.keys()},
+            **{k: v for k, v in params.items() if k in _params.keys()},
+        )
+        unit_metric_results_cache[cache_key] = (result, list(_inputs.keys()))
+
+    value = unit_metric_results_cache[cache_key][0]
+
+    if value_only:
+        return value
+
+    output_template = f"""
+    <table>
+        <thead>
+            <tr>
+                <th>Metric</th>
+                <th>Value</th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <td><strong>{test_id_to_name(metric_id)}</strong></td>
+                <td>{value:.4f}</td>
+            </tr>
+        </tbody>
+    </table>
+    <style>
+        th, td {{
+            padding: 5px;
+            text-align: left;
+        }}
+    </style>
+    """
+    result = _build_result(
+        results=value,
+        test_id=metric_id,
+        description="",
+        output_template=output_template,
+        inputs=unit_metric_results_cache[cache_key][1],
+    )
+
+    # in case the user tries to log the result object
+    def log(self):
+        raise Exception(
+            "Cannot log unit metrics directly..."
+            "You can run this unit metric as part of a composite metric and log that"
+        )
+
+    result.log = log
+
+    if show:
+        result.show()
 
     return result

validmind/unit_metrics/classification/sklearn/Accuracy.py

@@ -0,0 +1,14 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import accuracy_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def Accuracy(dataset, model):
+    """Calculates the accuracy of a model"""
+    return accuracy_score(dataset.y, dataset.y_pred(model))

validmind/unit_metrics/classification/sklearn/F1.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import f1_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def F1(model, dataset, **kwargs):
+    return f1_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/sklearn/Precision.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import precision_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def Precision(model, dataset, **kwargs):
+    return precision_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/sklearn/ROC_AUC.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import roc_auc_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def ROC_AUC(model, dataset, **kwargs):
+    return roc_auc_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/sklearn/Recall.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import recall_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def Recall(model, dataset, **kwargs):
+    return recall_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/composite.py

@@ -0,0 +1,228 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from dataclasses import dataclass
+from typing import List, Tuple, Union
+from uuid import uuid4
+
+from ..logging import get_logger
+from ..tests.decorator import _inspect_signature
+from ..utils import run_async, test_id_to_name
+from ..vm_models.test.metric import Metric
+from ..vm_models.test.metric_result import MetricResult
+from ..vm_models.test.result_summary import ResultSummary, ResultTable
+from ..vm_models.test.result_wrapper import MetricResultWrapper
+from . import load_metric, run_metric
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class CompositeMetric(Metric):
+    unit_metrics: List[str] = None
+
+    def __post_init__(self):
+        if self._unit_metrics:
+            self.unit_metrics = self._unit_metrics
+        elif self.unit_metrics is None:
+            raise ValueError("unit_metrics must be provided")
+
+        if hasattr(self, "_output_template") and self._output_template:
+            self.output_template = self._output_template
+
+    def run(self):
+        self.result = run_metrics(
+            test_id=self.test_id,
+            metric_ids=self.unit_metrics,
+            description=self.description(),
+            inputs=self._get_input_dict(),
+            params=self.params,
+            output_template=self.output_template,
+            show=False,
+        )
+
+        return self.result
+
+    def summary(self, result: dict):
+        return ResultSummary(results=[ResultTable(data=[result])])
+
+
+def load_composite_metric(
+    test_id: str = None,
+    metric_name: str = None,
+    unit_metrics: List[str] = None,
+    output_template: str = None,
+) -> Tuple[Union[None, str], Union[CompositeMetric, None]]:
+    # this function can either create a composite metric from a list of unit metrics or
+    # load a stored composite metric based on the test id
+
+    # TODO: figure out this circular import thing:
+    from ..api_client import get_metadata
+
+    if test_id:
+        # get the unit metric ids and output template (if any) from the metadata
+        try:
+            unit_metrics = run_async(
+                get_metadata, f"composite_metric_def:{test_id}:unit_metrics"
+            )["json"]
+            output_template = run_async(
+                get_metadata, f"composite_metric_def:{test_id}:output_template"
+            )["json"]["output_template"]
+        except Exception:
+            return f"Could not load composite metric {test_id}", None
+
+    description = f"""
+    Composite metric built from the following unit metrics:
+    {', '.join([metric_id.split('.')[-1] for metric_id in unit_metrics])}
+    """
+
+    class_def = type(
+        test_id.split(".")[-1] if test_id else metric_name,
+        (CompositeMetric,),
+        {
+            "__doc__": description,
+            "_unit_metrics": unit_metrics,
+            "_output_template": output_template,
+        },
+    )
+
+    required_inputs = set()
+    for metric_id in unit_metrics:
+        inputs, _ = _inspect_signature(load_metric(metric_id))
+        required_inputs.update(inputs.keys())
+
+    class_def.required_inputs = list(required_inputs)
+
+    return None, class_def
+
+
+def run_metrics(
+    name: str = None,
+    metric_ids: List[str] = None,
+    description: str = None,
+    output_template: str = None,
+    inputs: dict = None,
+    params: dict = None,
+    test_id: str = None,
+    show: bool = True,
+) -> MetricResultWrapper:
+    """Run a composite metric
+
+    Composite metrics are metrics that are composed of multiple unit metrics. This
+    works by running individual unit metrics and then combining the results into a
+    single "MetricResult" object that can be logged and displayed just like any other
+    metric result. The special thing about composite metrics is that when they are
+    logged to the platform, metadata describing the unit metrics and output template
+    used to generate the composite metric is also logged. This means that by grabbing
+    the metadata for a composite metric (identified by the test ID
+    `validmind.composite_metric.<name>`) the framework can rebuild and rerun it at
+    any time.
+
+    Args:
+        name (str, optional): Name of the composite metric. Required if test_id is not
+            provided. Defaults to None.
+        metric_ids (list[str]): List of unit metric IDs to run. Required.
+        description (str, optional): Description of the composite metric. Defaults to
+            None.
+        output_template (_type_, optional): Output template to customize the result
+            table.
+        inputs (_type_, optional): Inputs to pass to the unit metrics. Defaults to None
+        params (_type_, optional): Parameters to pass to the unit metrics. Defaults to
+            None.
+        test_id (str, optional): Test ID of the composite metric. Required if name is
+            not provided. Defaults to None.
+        show (bool, optional): Whether to show the result immediately. Defaults to True
+
+    Raises:
+        ValueError: If metric_ids is not provided
+        ValueError: If name or key is not provided
+
+    Returns:
+        MetricResultWrapper: The result wrapper object
+    """
+    if not metric_ids:
+        raise ValueError("metric_ids must be provided")
+
+    if not name and not test_id:
+        raise ValueError("name or key must be provided")
+
+    # if name is provided, make sure to squash it into a camel case string
+    if name:
+        name = "".join(word[0].upper() + word[1:] for word in name.split())
+
+    results = {}
+
+    for metric_id in metric_ids:
+        metric_name = test_id_to_name(metric_id)
+        results[metric_name] = run_metric(
+            metric_id=metric_id,
+            inputs=inputs,
+            params=params,
+            show=False,
+            value_only=True,
+        )
+
+    test_id = f"validmind.composite_metric.{name}" if not test_id else test_id
+
+    if not output_template:
+
+        def row(name):
+            return f"""
+            <tr>
+                <td><strong>{name}</strong></td>
+                <td>{{{{ value['{name}'] | number }}}}</td>
+            </tr>
+            """
+
+        output_template = f"""
+        <h1{test_id_to_name(test_id)}</h1>
+        <table>
+            <thead>
+                <tr>
+                    <th>Metric</th>
+                    <th>Value</th>
+                </tr>
+            </thead>
+            <tbody>
+                {"".join([row(name) for name in results.keys()])}
+            </tbody>
+        </table>
+        <style>
+            th, td {{
+                padding: 5px;
+                text-align: left;
+            }}
+        </style>
+        """
+
+    result_wrapper = MetricResultWrapper(
+        result_id=test_id,
+        result_metadata=[
+            {
+                "content_id": f"metric_description:{test_id}",
+                "text": description,
+            },
+            {
+                "content_id": f"composite_metric_def:{test_id}:unit_metrics",
+                "json": metric_ids,
+            },
+            {
+                "content_id": f"composite_metric_def:{test_id}:output_template",
+                "json": {"output_template": output_template},
+            },
+        ],
+        inputs=list(inputs.keys()),
+        output_template=output_template,
+        metric=MetricResult(
+            key=test_id,
+            ref_id=str(uuid4()),
+            value=results,
+            summary=ResultSummary(results=[ResultTable(data=[results])]),
+        ),
+    )
+
+    if show:
+        result_wrapper.show()
+
+    return result_wrapper

validmind/unit_metrics/regression/GiniCoefficient.py

@@ -0,0 +1,33 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+
+from validmind import tags, tasks
+
+
+@tags("regression", "unit_metric")
+@tasks("regression")
+def GiniCoefficient(dataset, model):
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Sort true values and corresponding predicted values
+    idx = np.argsort(y_true)
+    y_true_sorted = y_true[idx]
+    y_pred_sorted = y_pred[idx]
+
+    # Compute cumulative sums
+    cumsum_true = np.cumsum(y_true_sorted)
+    cumsum_pred = np.cumsum(y_pred_sorted)
+
+    # Normalize cumulative sums
+    cumsum_true_norm = cumsum_true / np.max(cumsum_true)
+    cumsum_pred_norm = cumsum_pred / np.max(cumsum_pred)
+
+    # Compute area under the Lorenz curve
+    area_lorenz = np.trapz(cumsum_pred_norm, x=cumsum_true_norm)
+
+    # Compute Gini coefficient
+    return 1 - 2 * area_lorenz

validmind/unit_metrics/regression/HuberLoss.py

@@ -0,0 +1,23 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+
+from validmind import tags, tasks
+
+
+@tags("regression", "unit_metric")
+@tasks("regression")
+def HuberLoss(model, dataset):
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # delta - Threshold for the squared error to be linear or quadratic.
+    delta = 1.0
+    error = y_true - y_pred
+
+    quadratic_part = np.minimum(np.abs(error), delta)
+    linear_part = np.abs(error) - quadratic_part
+
+    return np.mean(0.5 * quadratic_part**2 + delta * linear_part)

validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py

@@ -0,0 +1,30 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+
+from validmind import tags, tasks
+
+
+@tags("regression", "unit_metric")
+@tasks("regression")
+def KolmogorovSmirnovStatistic(dataset, model):
+    y_true = dataset.y.flatten()
+    y_pred = dataset.y_pred(model)
+
+    # Sort true values and corresponding predicted values
+    idx_true = np.argsort(y_true)
+    idx_pred = np.argsort(y_pred)
+    y_true_sorted = y_true[idx_true]
+    y_pred_sorted = y_pred[idx_pred]
+
+    # Compute cumulative distribution functions (CDFs)
+    cdf_true = np.arange(1, len(y_true_sorted) + 1) / len(y_true_sorted)
+    cdf_pred = np.arange(1, len(y_pred_sorted) + 1) / len(y_pred_sorted)
+
+    # Compute absolute differences between CDFs
+    diff_cdf = np.abs(cdf_true - cdf_pred)
+
+    # Find maximum absolute difference
+    return np.max(diff_cdf)

validmind/unit_metrics/regression/MeanAbsolutePercentageError.py

@@ -0,0 +1,16 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+
+from validmind import tags, tasks
+
+
+@tags("regression", "unit_metric")
+@tasks("regression")
+def MeanAbsolutePercentageError(model, dataset):
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

validmind/unit_metrics/regression/MeanBiasDeviation.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+
+from validmind import tags, tasks
+
+
+@tags("regression", "unit_metric")
+@tasks("regression")
+def MeanBiasDeviation(model, dataset):
+    return np.mean(dataset.y - dataset.y_pred(model))

validmind/unit_metrics/regression/QuantileLoss.py

@@ -0,0 +1,15 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+
+from validmind import tags, tasks
+
+
+@tags("regression", "unit_metric")
+@tasks("regression")
+def QuantileLoss(model, dataset, quantile=0.5):
+    error = dataset.y - dataset.y_pred(model)
+
+    return np.mean(np.maximum(quantile * error, (quantile - 1) * error))

validmind/unit_metrics/regression/sklearn/AdjustedRSquaredScore.py

@@ -0,0 +1,21 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import r2_score as _r2_score
+
+from validmind import tags, tasks
+
+
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def AdjustedRSquaredScore(model, dataset):
+    r2_score = _r2_score(
+        dataset.y,
+        dataset.y_pred(model),
+    )
+
+    row_count = len(dataset.y)
+    feature_count = len(dataset.get_features_columns())
+
+    return 1 - (1 - r2_score) * (row_count - 1) / (row_count - feature_count)

validmind/unit_metrics/regression/sklearn/MeanAbsoluteError.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import mean_absolute_error as _mean_absolute_error
+
+from validmind import tags, tasks
+
+
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def MeanAbsoluteError(model, dataset, **kwargs):
+    return _mean_absolute_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/sklearn/MeanSquaredError.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import mean_squared_error
+
+from validmind import tags, tasks
+
+
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def MeanSquaredError(model, dataset, **kwargs):
+    return mean_squared_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/sklearn/RSquaredScore.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import r2_score
+
+from validmind import tags, tasks
+
+
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def RSquaredError(model, dataset):
+    return r2_score(dataset.y, dataset.y_pred(model))