validmind 2.0.7__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/unit_metrics/classification/sklearn/F1.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import f1_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def F1(model, dataset, **kwargs):
+    return f1_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/sklearn/Precision.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import precision_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def Precision(model, dataset, **kwargs):
+    return precision_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/sklearn/ROC_AUC.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import roc_auc_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def ROC_AUC(model, dataset, **kwargs):
+    return roc_auc_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/classification/sklearn/Recall.py

@@ -0,0 +1,13 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.metrics import recall_score
+
+from validmind import tags, tasks
+
+
+@tags("classification", "sklearn", "unit_metric")
+@tasks("classification")
+def Recall(model, dataset, **kwargs):
+    return recall_score(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/composite.py

@@ -2,75 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import ast
-import inspect
 from dataclasses import dataclass
-from typing import List
+from typing import List, Tuple, Union
 from uuid import uuid4
 
-from ..errors import LoadTestError
 from ..logging import get_logger
-from ..utils import clean_docstring, run_async, test_id_to_name
+from ..tests.decorator import _inspect_signature
+from ..utils import run_async, test_id_to_name
 from ..vm_models.test.metric import Metric
 from ..vm_models.test.metric_result import MetricResult
 from ..vm_models.test.result_summary import ResultSummary, ResultTable
 from ..vm_models.test.result_wrapper import MetricResultWrapper
-from . import _get_metric_class, run_metric
+from . import load_metric, run_metric
 
 logger = get_logger(__name__)
 
 
-def _extract_class_methods(cls):
-    source = inspect.getsource(cls)
-    tree = ast.parse(source)
-
-    class MethodVisitor(ast.NodeVisitor):
-        def __init__(self):
-            self.methods = {}
-
-        def visit_FunctionDef(self, node):
-            self.methods[node.name] = node
-            self.generic_visit(node)
-
-    visitor = MethodVisitor()
-    visitor.visit(tree)
-
-    return visitor.methods
-
-
-def _extract_required_inputs(cls):
-    methods = _extract_class_methods(cls)
-
-    class Visitor(ast.NodeVisitor):
-        def __init__(self):
-            self.properties = set()
-            self.visited_methods = set()
-
-        def visit_Attribute(self, node):
-            if isinstance(node.value, ast.Attribute) and node.value.attr == "inputs":
-                self.properties.add(node.attr)
-
-            self.generic_visit(node)
-
-        def visit_Call(self, node):
-            if isinstance(node.func, ast.Attribute) and isinstance(
-                node.func.value, ast.Name
-            ):
-                if node.func.value.id == "self" and node.func.attr in methods:
-                    method_name = node.func.attr
-
-                    if method_name not in self.visited_methods:
-                        self.visited_methods.add(method_name)
-                        self.visit(methods[method_name])
-
-            self.generic_visit(node)
-
-    visitor = Visitor()
-    visitor.visit(methods["run"])
-
-    return visitor.properties
-
-
 @dataclass
 class CompositeMetric(Metric):
     unit_metrics: List[str] = None
@@ -106,7 +53,7 @@ def load_composite_metric(
     metric_name: str = None,
     unit_metrics: List[str] = None,
     output_template: str = None,
-) -> CompositeMetric:
+) -> Tuple[Union[None, str], Union[CompositeMetric, None]]:
     # this function can either create a composite metric from a list of unit metrics or
     # load a stored composite metric based on the test id
 
@@ -123,8 +70,7 @@ def load_composite_metric(
                 get_metadata, f"composite_metric_def:{test_id}:output_template"
             )["json"]["output_template"]
         except Exception:
-            logger.error(f"Could not load composite metric {test_id}")
-            raise LoadTestError(f"Could not load composite metric {test_id}")
+            return f"Could not load composite metric {test_id}", None
 
     description = f"""
     Composite metric built from the following unit metrics:
@@ -143,13 +89,12 @@ def load_composite_metric(
 
     required_inputs = set()
     for metric_id in unit_metrics:
-        metric_cls = _get_metric_class(metric_id)
-        # required_inputs.update(_extract_required_inputs(metric_cls))
-        required_inputs.update(metric_cls.required_inputs or [])
+        inputs, _ = _inspect_signature(load_metric(metric_id))
+        required_inputs.update(inputs.keys())
 
     class_def.required_inputs = list(required_inputs)
 
-    return class_def
+    return None, class_def
 
 
 def run_metrics(
@@ -209,22 +154,24 @@ def run_metrics(
     results = {}
 
     for metric_id in metric_ids:
-        result = run_metric(
+        metric_name = test_id_to_name(metric_id)
+        results[metric_name] = run_metric(
             metric_id=metric_id,
             inputs=inputs,
             params=params,
+            show=False,
+            value_only=True,
         )
-        results[list(result.summary.keys())[0]] = result.value
 
     test_id = f"validmind.composite_metric.{name}" if not test_id else test_id
 
     if not output_template:
 
-        def row(key):
+        def row(name):
             return f"""
             <tr>
-                <td><strong>{key.upper()}</strong></td>
-                <td>{{{{ value['{key}'] | number }}}}</td>
+                <td><strong>{name}</strong></td>
+                <td>{{{{ value['{name}'] | number }}}}</td>
             </tr>
             """
 
@@ -238,9 +185,15 @@ def run_metrics(
                 </tr>
             </thead>
             <tbody>
-                {"".join([row(key) for key in results.keys()])}
+                {"".join([row(name) for name in results.keys()])}
             </tbody>
         </table>
+        <style>
+            th, td {{
+                padding: 5px;
+                text-align: left;
+            }}
+        </style>
         """
 
     result_wrapper = MetricResultWrapper(
@@ -248,7 +201,7 @@ def run_metrics(
         result_metadata=[
             {
                 "content_id": f"metric_description:{test_id}",
-                "text": clean_docstring(description),
+                "text": description,
             },
             {
                 "content_id": f"composite_metric_def:{test_id}:unit_metrics",

validmind/unit_metrics/regression/GiniCoefficient.py

@@ -2,38 +2,32 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class GiniCoefficient(UnitMetric):
-    required_inputs = ["dataset", "model"]
+from validmind import tags, tasks
 
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
 
-        # Sort true values and corresponding predicted values
-        idx = np.argsort(y_true)
-        y_true_sorted = y_true[idx]
-        y_pred_sorted = y_pred[idx]
+@tags("regression", "unit_metric")
+@tasks("regression")
+def GiniCoefficient(dataset, model):
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
 
-        # Compute cumulative sums
-        cumsum_true = np.cumsum(y_true_sorted)
-        cumsum_pred = np.cumsum(y_pred_sorted)
+    # Sort true values and corresponding predicted values
+    idx = np.argsort(y_true)
+    y_true_sorted = y_true[idx]
+    y_pred_sorted = y_pred[idx]
 
-        # Normalize cumulative sums
-        cumsum_true_norm = cumsum_true / np.max(cumsum_true)
-        cumsum_pred_norm = cumsum_pred / np.max(cumsum_pred)
+    # Compute cumulative sums
+    cumsum_true = np.cumsum(y_true_sorted)
+    cumsum_pred = np.cumsum(y_pred_sorted)
 
-        # Compute area under the Lorenz curve
-        area_lorenz = np.trapz(cumsum_pred_norm, x=cumsum_true_norm)
+    # Normalize cumulative sums
+    cumsum_true_norm = cumsum_true / np.max(cumsum_true)
+    cumsum_pred_norm = cumsum_pred / np.max(cumsum_pred)
 
-        # Compute Gini coefficient
-        gini_coeff = 1 - 2 * area_lorenz
+    # Compute area under the Lorenz curve
+    area_lorenz = np.trapz(cumsum_pred_norm, x=cumsum_true_norm)
 
-        return self.cache_results(metric_value=gini_coeff)
+    # Compute Gini coefficient
+    return 1 - 2 * area_lorenz

validmind/unit_metrics/regression/HuberLoss.py

@@ -2,26 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 
-from validmind.vm_models import UnitMetric
+from validmind import tags, tasks
 
 
-@dataclass
-class HuberLoss(UnitMetric):
-    required_inputs = ["dataset", "model"]
+@tags("regression", "unit_metric")
+@tasks("regression")
+def HuberLoss(model, dataset):
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
 
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
+    # delta - Threshold for the squared error to be linear or quadratic.
+    delta = 1.0
+    error = y_true - y_pred
 
-        # delta - Threshold for the squared error to be linear or quadratic.
-        delta = 1.0
-        error = y_true - y_pred
-        quadratic_part = np.minimum(np.abs(error), delta)
-        linear_part = np.abs(error) - quadratic_part
-        value = np.mean(0.5 * quadratic_part**2 + delta * linear_part)
+    quadratic_part = np.minimum(np.abs(error), delta)
+    linear_part = np.abs(error) - quadratic_part
 
-        return self.cache_results(metric_value=value)
+    return np.mean(0.5 * quadratic_part**2 + delta * linear_part)

validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py

@@ -2,35 +2,29 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class KolmogorovSmirnovStatistic(UnitMetric):
-    required_inputs = ["dataset", "model"]
+from validmind import tags, tasks
 
-    def run(self):
-        y_true = self.inputs.dataset.y.flatten()
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
 
-        # Sort true values and corresponding predicted values
-        idx_true = np.argsort(y_true)
-        idx_pred = np.argsort(y_pred)
-        y_true_sorted = y_true[idx_true]
-        y_pred_sorted = y_pred[idx_pred]
+@tags("regression", "unit_metric")
+@tasks("regression")
+def KolmogorovSmirnovStatistic(dataset, model):
+    y_true = dataset.y.flatten()
+    y_pred = dataset.y_pred(model)
 
-        # Compute cumulative distribution functions (CDFs)
-        cdf_true = np.arange(1, len(y_true_sorted) + 1) / len(y_true_sorted)
-        cdf_pred = np.arange(1, len(y_pred_sorted) + 1) / len(y_pred_sorted)
+    # Sort true values and corresponding predicted values
+    idx_true = np.argsort(y_true)
+    idx_pred = np.argsort(y_pred)
+    y_true_sorted = y_true[idx_true]
+    y_pred_sorted = y_pred[idx_pred]
 
-        # Compute absolute differences between CDFs
-        diff_cdf = np.abs(cdf_true - cdf_pred)
+    # Compute cumulative distribution functions (CDFs)
+    cdf_true = np.arange(1, len(y_true_sorted) + 1) / len(y_true_sorted)
+    cdf_pred = np.arange(1, len(y_pred_sorted) + 1) / len(y_pred_sorted)
 
-        # Find maximum absolute difference
-        ks_statistic = np.max(diff_cdf)
+    # Compute absolute differences between CDFs
+    diff_cdf = np.abs(cdf_true - cdf_pred)
 
-        return self.cache_results(metric_value=ks_statistic)
+    # Find maximum absolute difference
+    return np.max(diff_cdf)

validmind/unit_metrics/regression/MeanAbsolutePercentageError.py

@@ -2,21 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class MeanAbsolutePercentageError(UnitMetric):
-    required_inputs = ["dataset", "model"]
+from validmind import tags, tasks
 
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
 
-        value = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+@tags("regression", "unit_metric")
+@tasks("regression")
+def MeanAbsolutePercentageError(model, dataset):
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
 
-        return self.cache_results(metric_value=value)
+    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

validmind/unit_metrics/regression/MeanBiasDeviation.py

@@ -2,21 +2,12 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class MeanBiasDeviation(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
+from validmind import tags, tasks
 
-        value = np.mean(y_pred - y_true)
 
-        return self.cache_results(metric_value=value)
+@tags("regression", "unit_metric")
+@tasks("regression")
+def MeanBiasDeviation(model, dataset):
+    return np.mean(dataset.y - dataset.y_pred(model))

validmind/unit_metrics/regression/QuantileLoss.py

@@ -2,24 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class QuantileLoss(UnitMetric):
-    required_inputs = ["dataset", "model"]
+from validmind import tags, tasks
 
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
 
-        error = y_true - y_pred
-        # Quantile value (between 0 and 1).
-        quantile = 0.5
-        value = np.mean(np.maximum(quantile * error, (quantile - 1) * error))
+@tags("regression", "unit_metric")
+@tasks("regression")
+def QuantileLoss(model, dataset, quantile=0.5):
+    error = dataset.y - dataset.y_pred(model)
 
-        return self.cache_results(metric_value=value)
+    return np.mean(np.maximum(quantile * error, (quantile - 1) * error))

validmind/unit_metrics/regression/sklearn/AdjustedRSquaredScore.py

@@ -2,26 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from sklearn.metrics import r2_score as _r2_score
 
-import sklearn.metrics as metrics
+from validmind import tags, tasks
 
-from validmind.vm_models import UnitMetric
 
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def AdjustedRSquaredScore(model, dataset):
+    r2_score = _r2_score(
+        dataset.y,
+        dataset.y_pred(model),
+    )
 
-@dataclass
-class AdjustedRSquaredScore(UnitMetric):
-    required_inputs = ["dataset", "model"]
+    row_count = len(dataset.y)
+    feature_count = len(dataset.get_features_columns())
 
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        X_columns = self.inputs.dataset.get_features_columns()
-        row_count = len(y_true)
-        feature_count = len(X_columns)
-        value = 1 - (1 - metrics.r2_score(y_true, y_pred)) * (row_count - 1) / (
-            row_count - feature_count
-        )
-
-        return self.cache_results(metric_value=value)
+    return 1 - (1 - r2_score) * (row_count - 1) / (row_count - feature_count)

validmind/unit_metrics/regression/sklearn/MeanAbsoluteError.py

@@ -2,21 +2,12 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from sklearn.metrics import mean_absolute_error as _mean_absolute_error
 
-from sklearn.metrics import mean_absolute_error
+from validmind import tags, tasks
 
-from validmind.vm_models import UnitMetric
 
-
-@dataclass
-class MeanAbsoluteError(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = mean_absolute_error(y_true, y_pred, **self.params)
-
-        return self.cache_results(metric_value=value)
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def MeanAbsoluteError(model, dataset, **kwargs):
+    return _mean_absolute_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/sklearn/MeanSquaredError.py

@@ -2,21 +2,12 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 from sklearn.metrics import mean_squared_error
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class MeanSquaredError(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
+from validmind import tags, tasks
 
-        value = mean_squared_error(y_true, y_pred, **self.params)
 
-        return self.cache_results(metric_value=value)
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def MeanSquaredError(model, dataset, **kwargs):
+    return mean_squared_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/sklearn/RSquaredScore.py

@@ -2,21 +2,12 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from sklearn.metrics import r2_score
 
-import sklearn.metrics as metrics
+from validmind import tags, tasks
 
-from validmind.vm_models import UnitMetric
 
-
-@dataclass
-class RSquaredScore(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
-
-        value = metrics.r2_score(y_true, y_pred)
-
-        return self.cache_results(metric_value=value)
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def RSquaredError(model, dataset):
+    return r2_score(dataset.y, dataset.y_pred(model))

validmind/unit_metrics/regression/sklearn/RootMeanSquaredError.py

@@ -2,22 +2,19 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 from sklearn.metrics import mean_squared_error
 
-from validmind.vm_models import UnitMetric
-
-
-@dataclass
-class RootMeanSquaredError(UnitMetric):
-    required_inputs = ["dataset", "model"]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(model_id=self.inputs.model.input_id)
+from validmind import tags, tasks
 
-        value = np.sqrt(mean_squared_error(y_true, y_pred, **self.params))
 
-        return self.cache_results(metric_value=value)
+@tags("regression", "sklearn", "unit_metric")
+@tasks("regression")
+def RootMeanSquaredError(model, dataset, **kwargs):
+    return np.sqrt(
+        mean_squared_error(
+            dataset.y,
+            dataset.y_pred(model),
+            **kwargs,
+        )
+    )