validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/unit_metrics/composite.py

@@ -1,238 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-from typing import List, Tuple, Union
-from uuid import uuid4
-
-from ..ai.test_descriptions import get_description_metadata
-from ..logging import get_logger
-from ..tests.decorator import _inspect_signature
-from ..utils import run_async, test_id_to_name
-from ..vm_models.test.metric import Metric
-from ..vm_models.test.metric_result import MetricResult
-from ..vm_models.test.result_summary import ResultSummary, ResultTable
-from ..vm_models.test.result_wrapper import MetricResultWrapper
-from . import load_metric, run_metric
-
-logger = get_logger(__name__)
-
-
-@dataclass
-class CompositeMetric(Metric):
-    unit_metrics: List[str] = None
-
-    def __post_init__(self):
-        if self._unit_metrics:
-            self.unit_metrics = self._unit_metrics
-        elif self.unit_metrics is None:
-            raise ValueError("unit_metrics must be provided")
-
-        if hasattr(self, "_output_template") and self._output_template:
-            self.output_template = self._output_template
-
-    def run(self):
-        self.result = run_metrics(
-            test_id=self.test_id,
-            metric_ids=self.unit_metrics,
-            description=self.description(),
-            inputs=self._get_input_dict(),
-            accessed_inputs=self.get_accessed_inputs(),
-            params=self.params,
-            output_template=self.output_template,
-            show=False,
-            generate_description=self.generate_description,
-        )
-
-        return self.result
-
-    def summary(self, result: dict):
-        return ResultSummary(results=[ResultTable(data=[result])])
-
-
-def load_composite_metric(
-    test_id: str = None,
-    metric_name: str = None,
-    unit_metrics: List[str] = None,
-    output_template: str = None,
-) -> Tuple[Union[None, str], Union[CompositeMetric, None]]:
-    # this function can either create a composite metric from a list of unit metrics or
-    # load a stored composite metric based on the test id
-
-    # TODO: figure out this circular import thing:
-    from ..api_client import get_metadata
-
-    if test_id:
-        # get the unit metric ids and output template (if any) from the metadata
-        try:
-            unit_metrics = run_async(
-                get_metadata, f"composite_metric_def:{test_id}:unit_metrics"
-            )["json"]
-            output_template = run_async(
-                get_metadata, f"composite_metric_def:{test_id}:output_template"
-            )["json"]["output_template"]
-        except Exception:
-            return f"Could not load composite metric {test_id}", None
-
-    description = f"""
-    Composite metric built from the following unit metrics:
-    {', '.join([metric_id.split('.')[-1] for metric_id in unit_metrics])}
-    """
-
-    class_def = type(
-        test_id.split(".")[-1] if test_id else metric_name,
-        (CompositeMetric,),
-        {
-            "__doc__": description,
-            "_unit_metrics": unit_metrics,
-            "_output_template": output_template,
-        },
-    )
-
-    required_inputs = set()
-    for metric_id in unit_metrics:
-        inputs, _ = _inspect_signature(load_metric(metric_id))
-        required_inputs.update(inputs.keys())
-
-    class_def.required_inputs = list(required_inputs)
-
-    return None, class_def
-
-
-def run_metrics(
-    name: str = None,
-    metric_ids: List[str] = None,
-    description: str = None,
-    output_template: str = None,
-    inputs: dict = None,
-    accessed_inputs: List[str] = None,
-    params: dict = None,
-    test_id: str = None,
-    show: bool = True,
-    generate_description: bool = True,
-) -> MetricResultWrapper:
-    """Run a composite metric
-
-    Composite metrics are metrics that are composed of multiple unit metrics. This
-    works by running individual unit metrics and then combining the results into a
-    single "MetricResult" object that can be logged and displayed just like any other
-    metric result. The special thing about composite metrics is that when they are
-    logged to the platform, metadata describing the unit metrics and output template
-    used to generate the composite metric is also logged. This means that by grabbing
-    the metadata for a composite metric (identified by the test ID
-    `validmind.composite_metric.<name>`) the framework can rebuild and rerun it at
-    any time.
-
-    Args:
-        name (str, optional): Name of the composite metric. Required if test_id is not
-            provided. Defaults to None.
-        metric_ids (list[str]): List of unit metric IDs to run. Required.
-        description (str, optional): Description of the composite metric. Defaults to
-            None.
-        output_template (_type_, optional): Output template to customize the result
-            table.
-        inputs (_type_, optional): Inputs to pass to the unit metrics. Defaults to None
-        accessed_inputs (_type_, optional): Inputs that were accessed when running the
-            unit metrics - used for input tracking. Defaults to None.
-        params (_type_, optional): Parameters to pass to the unit metrics. Defaults to
-            None.
-        test_id (str, optional): Test ID of the composite metric. Required if name is
-            not provided. Defaults to None.
-        show (bool, optional): Whether to show the result immediately. Defaults to True
-
-    Raises:
-        ValueError: If metric_ids is not provided
-        ValueError: If name or key is not provided
-
-    Returns:
-        MetricResultWrapper: The result wrapper object
-    """
-    if not metric_ids:
-        raise ValueError("metric_ids must be provided")
-
-    if not name and not test_id:
-        raise ValueError("name or key must be provided")
-
-    # if name is provided, make sure to squash it into a camel case string
-    if name:
-        name = "".join(word[0].upper() + word[1:] for word in name.split())
-
-    results = {}
-
-    for metric_id in metric_ids:
-        metric_name = test_id_to_name(metric_id)
-        results[metric_name] = run_metric(
-            metric_id=metric_id,
-            inputs=inputs,
-            params=params,
-            show=False,
-            value_only=True,
-        )
-
-    test_id = f"validmind.composite_metric.{name}" if not test_id else test_id
-
-    if not output_template:
-
-        def row(name):
-            return f"""
-            <tr>
-                <td><strong>{name}</strong></td>
-                <td>{{{{ value['{name}'] | number }}}}</td>
-            </tr>
-            """
-
-        output_template = f"""
-        <h1{test_id_to_name(test_id)}</h1>
-        <table>
-            <thead>
-                <tr>
-                    <th>Metric</th>
-                    <th>Value</th>
-                </tr>
-            </thead>
-            <tbody>
-                {"".join([row(name) for name in results.keys()])}
-            </tbody>
-        </table>
-        <style>
-            th, td {{
-                padding: 5px;
-                text-align: left;
-            }}
-        </style>
-        """
-
-    result_summary = ResultSummary(results=[ResultTable(data=[results])])
-    result_wrapper = MetricResultWrapper(
-        result_id=test_id,
-        result_metadata=[
-            get_description_metadata(
-                test_id=test_id,
-                default_description=description,
-                summary=result_summary.serialize(),
-                should_generate=generate_description,
-            ),
-            {
-                "content_id": f"composite_metric_def:{test_id}:unit_metrics",
-                "json": metric_ids,
-            },
-            {
-                "content_id": f"composite_metric_def:{test_id}:output_template",
-                "json": {"output_template": output_template},
-            },
-        ],
-        inputs=accessed_inputs,
-        output_template=output_template,
-        metric=MetricResult(
-            key=test_id,
-            ref_id=str(uuid4()),
-            value=results,
-            summary=result_summary,
-        ),
-    )
-
-    if show:
-        result_wrapper.show()
-
-    return result_wrapper

validmind/vm_models/test/metric.py

@@ -1,98 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-"""
-Class for storing ValidMind metric objects and associated
-data for display and reporting purposes
-"""
-from abc import abstractmethod
-from dataclasses import dataclass
-from typing import ClassVar, List, Optional, Union
-
-import pandas as pd
-
-from ...ai.test_descriptions import get_description_metadata
-from ...errors import MissingCacheResultsArgumentsError
-from ..figure import Figure
-from .metric_result import MetricResult
-from .result_wrapper import MetricResultWrapper
-from .test import Test
-
-
-@dataclass
-class Metric(Test):
-    """
-    Metric objects track the schema supported by the ValidMind API
-    """
-
-    # Class Variables
-    test_type: ClassVar[str] = "Metric"
-
-    type: ClassVar[str] = ""  # type of metric: "training", "evaluation", etc.
-    scope: ClassVar[str] = ""  # scope of metric: "training_dataset"
-    value_formatter: ClassVar[Optional[str]] = None  # "records" or "key_values"
-
-    # Instance Variables
-    result: MetricResultWrapper = None  # populated by cache_results() method
-
-    @abstractmethod
-    def summary(self, metric_value: Optional[Union[dict, list, pd.DataFrame]] = None):
-        """
-        Return the metric summary. Should be overridden by subclasses. Defaults to None.
-        The metric summary allows renderers (e.g. Word and ValidMind UI) to display a
-        short summary of the metric results.
-
-        We return None here because the metric summary is optional.
-        """
-        return None
-
-    def cache_results(
-        self,
-        metric_value: Optional[Union[dict, list, pd.DataFrame]] = None,
-        figures: Optional[List[Figure]] = None,
-    ):
-        """
-        Cache the results of the metric calculation and do any post-processing if needed
-
-        Args:
-            metric_value (Union[dict, list, pd.DataFrame]): The value of the metric
-            figures (Optional[object]): Any figures to attach to the test suite result
-
-        Returns:
-            TestSuiteResult: The test suite result object
-        """
-        if metric_value is None and figures is None:
-            raise MissingCacheResultsArgumentsError(
-                "Metric must provide a metric value or figures to cache_results"
-            )
-
-        metric = MetricResult(
-            key=self.test_id,
-            ref_id=self._ref_id,
-            value=metric_value if metric_value is not None else {},
-            value_formatter=self.value_formatter,
-            summary=self.summary(metric_value),
-        )
-
-        self.result = MetricResultWrapper(
-            result_id=self.test_id,
-            result_description=self.description(),
-            result_metadata=[
-                (
-                    get_description_metadata(
-                        test_id=self.test_id,
-                        default_description=self.description(),
-                        summary=metric.serialize()["summary"],
-                        figures=figures,
-                        should_generate=self.generate_description,
-                    )
-                )
-            ],
-            metric=metric,
-            figures=figures,
-            inputs=self.get_accessed_inputs(),
-            output_template=self.output_template,
-        )
-
-        return self.result

validmind/vm_models/test/metric_result.py

@@ -1,61 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-"""
-MetricResult wrapper
-"""
-
-
-from dataclasses import dataclass
-from typing import Optional, Union
-
-import pandas as pd
-
-from ...errors import InvalidValueFormatterError
-from ...utils import format_key_values, format_records
-from .result_summary import ResultSummary
-
-
-@dataclass
-class MetricResult:
-    """
-    MetricResult class definition. A MetricResult is returned by any internal method
-    that extracts metrics from a dataset or model, and returns 1) Metric and Figure
-    objects that can be sent to the API and 2) and plots and metadata for display purposes
-    """
-
-    key: dict
-    ref_id: str
-    value: Union[dict, list, pd.DataFrame]
-    summary: Optional[ResultSummary] = None
-    value_formatter: Optional[str] = None
-
-    def serialize(self):
-        """
-        Serializes the Metric to a dictionary so it can be sent to the API
-        """
-        if self.value_formatter == "records":
-            value = format_records(self.value)
-        elif self.value_formatter == "key_values":
-            value = format_key_values(self.value)
-        elif self.value_formatter is not None:
-            raise InvalidValueFormatterError(
-                f"Invalid value_formatter: {self.value_formatter}. "
-                "Must be one of 'records' or 'key_values'"
-            )
-        else:
-            # TODO: we need to handle formatting arbitrary shapes of data
-            value = self.value
-
-        if isinstance(value, pd.DataFrame):
-            raise InvalidValueFormatterError(
-                "A DataFrame value was provided but no value_formatter was specified."
-            )
-
-        return {
-            "key": self.key,
-            "ref_id": self.ref_id,
-            "value": value,
-            "summary": self.summary.serialize() if self.summary else None,
-        }

validmind/vm_models/test/output_template.py

@@ -1,55 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from datetime import datetime
-
-from dateutil import parser
-from jinja2 import Environment
-
-
-def format_date(value, format="%Y-%m-%d"):
-    if value is None:
-        return None
-
-    if isinstance(value, datetime):
-        return value.strftime(format)
-
-    return parser.parse(value).strftime(format)
-
-
-def format_number(value, format="{:,.4f}"):
-    return format.format(value)
-
-
-def _generate_empty_historical_data(value):
-    return [
-        {
-            "value": value,
-            "metadata": {
-                "created_at": format_date(datetime.now()),
-            },
-        }
-    ]
-
-
-class OutputTemplate:
-    def __init__(self, template_string, template_engine=None):
-        if template_engine is None:
-            template_engine = Environment()
-            template_engine.filters["date"] = format_date
-            template_engine.filters["number"] = format_number
-
-        self.template_engine = template_engine
-        self.template_string = template_string
-
-    def render(self, value, values_history=None):
-        template = self.template_engine.from_string(self.template_string)
-
-        if not values_history:
-            values_history = _generate_empty_historical_data(value)
-
-        return template.render(
-            value=value,
-            metric_history=values_history,
-        )

validmind/vm_models/test/result_summary.py

@@ -1,76 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-from typing import Any, List, Union
-
-import pandas as pd
-
-
-@dataclass
-class ResultTableMetadata:
-    """
-    A dataclass that holds the metadata of a table summary
-    """
-
-    title: str
-
-
-@dataclass
-class ResultTable:
-    """
-    A dataclass that holds the table summary of result
-    """
-
-    data: Union[List[Any], pd.DataFrame]
-    type: str = "table"
-    metadata: ResultTableMetadata = None
-
-    def serialize(self, as_df=False):
-        """
-        Serializes the Figure to a dictionary so it can be sent to the API.
-
-        This method accepts as_df parameter to return the data as a DataFrame
-        if we're returning the data to R.
-        """
-        table_result = {
-            "type": self.type,
-        }
-
-        # Convert to a DataFrame so that we can round the values in a standard way
-        table_df = pd.DataFrame(self.data) if isinstance(self.data, list) else self.data
-        table_df = table_df.round(4)
-
-        if as_df:
-            table_result["data"] = table_df
-        else:
-            table_result["data"] = table_df.to_dict(orient="records")
-
-        if self.metadata is not None:
-            table_result["metadata"] = vars(self.metadata)
-
-        return table_result
-
-
-@dataclass()
-class ResultSummary:
-    """
-    A dataclass that holds the summary of a metric or threshold test results
-    """
-
-    results: List[ResultTable]  # TBD other types of results
-
-    def add_result(self, result: ResultTable):
-        """
-        Adds a result to the list of results
-        """
-        if self.results is None:
-            self.results = []
-        self.results.append(result)
-
-    def serialize(self, as_df=False):
-        """
-        Serializes the ResultSummary to a list of results
-        """
-        return [result.serialize(as_df) for result in self.results]