genesis-flow 1.0.0__py3-none-any.whl

mlflow/models/evaluation/validation.py

@@ -0,0 +1,434 @@
+import logging
+import operator
+import os
+from decimal import Decimal
+from typing import Optional
+
+from mlflow.exceptions import MlflowException
+from mlflow.models.evaluation import EvaluationResult
+from mlflow.protos.databricks_pb2 import BAD_REQUEST, INVALID_PARAMETER_VALUE
+
+_logger = logging.getLogger(__name__)
+
+
+class MetricThreshold:
+    """
+    This class allows you to define metric thresholds for model validation.
+    Allowed thresholds are: threshold, min_absolute_change, min_relative_change.
+
+    Args:
+        threshold: (Optional) A number representing the value threshold for the metric.
+
+            - If greater is better for the metric, the metric value has to be
+              >= threshold to pass validation.
+            - Otherwise, the metric value has to be <= threshold to pass the validation.
+
+        min_absolute_change: (Optional) A positive number representing the minimum absolute
+            change required for candidate model to pass validation with
+            the baseline model.
+
+            - If greater is better for the metric, metric value has to be
+              >= baseline model metric value + min_absolute_change to pass the validation.
+            - Otherwise, metric value has to be <= baseline model metric value - min_absolute_change
+              to pass the validation.
+
+        min_relative_change: (Optional) A floating point number between 0 and 1 representing
+            the minimum relative change (in percentage of
+            baseline model metric value) for candidate model
+            to pass the comparison with the baseline model.
+
+            - If greater is better for the metric, metric value has to be
+              >= baseline model metric value * (1 + min_relative_change)
+            - Otherwise, metric value has to be
+              <= baseline model metric value * (1 - min_relative_change)
+            - Note that if the baseline model metric value is equal to 0, the
+              threshold falls back performing a simple verification that the
+              candidate metric value is better than the baseline metric value,
+              i.e. metric value >= baseline model metric value + 1e-10 if greater
+              is better; metric value <= baseline model metric value - 1e-10 if
+              lower is better.
+
+        greater_is_better: A required boolean representing whether greater value is
+            better for the metric.
+    """
+
+    def __init__(
+        self,
+        threshold=None,
+        min_absolute_change=None,
+        min_relative_change=None,
+        greater_is_better=None,
+    ):
+        if threshold is not None and type(threshold) not in {int, float}:
+            raise MetricThresholdClassException("`threshold` parameter must be a number.")
+        if min_absolute_change is not None and (
+            type(min_absolute_change) not in {int, float} or min_absolute_change <= 0
+        ):
+            raise MetricThresholdClassException(
+                "`min_absolute_change` parameter must be a positive number."
+            )
+        if min_relative_change is not None:
+            if not isinstance(min_relative_change, float):
+                raise MetricThresholdClassException(
+                    "`min_relative_change` parameter must be a floating point number."
+                )
+            if min_relative_change < 0 or min_relative_change > 1:
+                raise MetricThresholdClassException(
+                    "`min_relative_change` parameter must be between 0 and 1."
+                )
+        if greater_is_better is None:
+            raise MetricThresholdClassException("`greater_is_better` parameter must be defined.")
+        if not isinstance(greater_is_better, bool):
+            raise MetricThresholdClassException("`greater_is_better` parameter must be a boolean.")
+        if threshold is None and min_absolute_change is None and min_relative_change is None:
+            raise MetricThresholdClassException("no threshold was specified.")
+        self._threshold = threshold
+        self._min_absolute_change = min_absolute_change
+        self._min_relative_change = min_relative_change
+        self._greater_is_better = greater_is_better
+
+    @property
+    def threshold(self):
+        """
+        Value of the threshold.
+        """
+        return self._threshold
+
+    @property
+    def min_absolute_change(self):
+        """
+        Value of the minimum absolute change required to pass model comparison with baseline model.
+        """
+        return self._min_absolute_change
+
+    @property
+    def min_relative_change(self):
+        """
+        Float value of the minimum relative change required to pass model comparison with
+        baseline model.
+        """
+        return self._min_relative_change
+
+    @property
+    def greater_is_better(self):
+        """
+        Boolean value representing whether greater value is better for the metric.
+        """
+        return self._greater_is_better
+
+    def __str__(self):
+        """
+        Returns a human-readable string consisting of all specified thresholds.
+        """
+        threshold_strs = []
+        if self._threshold is not None:
+            threshold_strs.append(f"Threshold: {self._threshold}.")
+        if self._min_absolute_change is not None:
+            threshold_strs.append(f"Minimum Absolute Change: {self._min_absolute_change}.")
+        if self._min_relative_change is not None:
+            threshold_strs.append(f"Minimum Relative Change: {self._min_relative_change}.")
+        if self._greater_is_better is not None:
+            if self._greater_is_better:
+                threshold_strs.append("Greater value is better.")
+            else:
+                threshold_strs.append("Lower value is better.")
+        return " ".join(threshold_strs)
+
+
+class MetricThresholdClassException(MlflowException):
+    def __init__(self, _message, **kwargs):
+        message = "Could not instantiate MetricThreshold class: " + _message
+        super().__init__(message, error_code=INVALID_PARAMETER_VALUE, **kwargs)
+
+
+class _MetricValidationResult:
+    """
+    Internal class for representing validation result per metric.
+    Not user facing, used for organizing metric failures and generating failure message
+    more conveniently.
+
+    Args:
+        metric_name: String representing the metric name
+        candidate_metric_value: value of metric for candidate model
+        metric_threshold: :py:class: `MetricThreshold<mlflow.models.validation.MetricThreshold>`
+            The MetricThreshold for the metric.
+        baseline_metric_value: value of metric for baseline model
+    """
+
+    missing_candidate = False
+    missing_baseline = False
+    threshold_failed = False
+    min_absolute_change_failed = False
+    min_relative_change_failed = False
+
+    def __init__(
+        self,
+        metric_name,
+        candidate_metric_value,
+        metric_threshold,
+        baseline_metric_value=None,
+    ):
+        self.metric_name = metric_name
+        self.candidate_metric_value = candidate_metric_value
+        self.baseline_metric_value = baseline_metric_value
+        self.metric_threshold = metric_threshold
+
+    def __str__(self):
+        """
+        Returns a human-readable string representing the validation result for the metric.
+        """
+        if self.is_success():
+            return f"Metric {self.metric_name} passed the validation."
+
+        if self.missing_candidate:
+            return (
+                f"Metric validation failed: metric {self.metric_name} was missing from the "
+                f"evaluation result of the candidate model."
+            )
+
+        result_strs = []
+        if self.threshold_failed:
+            result_strs.append(
+                f"Metric {self.metric_name} value threshold check failed: "
+                f"candidate model {self.metric_name} = {self.candidate_metric_value}, "
+                f"{self.metric_name} threshold = {self.metric_threshold.threshold}."
+            )
+        if self.missing_baseline:
+            result_strs.append(
+                f"Model comparison failed: metric {self.metric_name} was missing from "
+                f"the evaluation result of the baseline model."
+            )
+        else:
+            if self.min_absolute_change_failed:
+                result_strs.append(
+                    f"Metric {self.metric_name} minimum absolute change check failed: "
+                    f"candidate model {self.metric_name} = {self.candidate_metric_value}, "
+                    f"baseline model {self.metric_name} = {self.baseline_metric_value}, "
+                    f"{self.metric_name} minimum absolute change threshold = "
+                    f"{self.metric_threshold.min_absolute_change}."
+                )
+            if self.min_relative_change_failed:
+                result_strs.append(
+                    f"Metric {self.metric_name} minimum relative change check failed: "
+                    f"candidate model {self.metric_name} = {self.candidate_metric_value}, "
+                    f"baseline model {self.metric_name} = {self.baseline_metric_value}, "
+                    f"{self.metric_name} minimum relative change threshold = "
+                    f"{self.metric_threshold.min_relative_change}."
+                )
+        return " ".join(result_strs)
+
+    def is_success(self):
+        return (
+            not self.missing_candidate
+            and not self.missing_baseline
+            and not self.threshold_failed
+            and not self.min_absolute_change_failed
+            and not self.min_relative_change_failed
+        )
+
+
+class ModelValidationFailedException(MlflowException):
+    def __init__(self, message, **kwargs):
+        super().__init__(message, error_code=BAD_REQUEST, **kwargs)
+
+
+def validate_evaluation_results(
+    validation_thresholds: dict[str, MetricThreshold],
+    candidate_result: EvaluationResult,
+    baseline_result: Optional[EvaluationResult] = None,
+):
+    """
+    Validate the evaluation result from one model (candidate) against another
+    model (baseline). If the candidate results do not meet the validation
+    thresholds, an ModelValidationFailedException will be raised.
+
+    .. note::
+
+        This API is a replacement for the deprecated model validation
+        functionality in the :py:func:`mlflow.evaluate` API.
+
+    Args:
+        validation_thresholds: A dictionary of metric name to
+            :py:class:`mlflow.models.MetricThreshold` used for model validation.
+            Each metric name must either be the name of a builtin metric or the
+            name of a metric defined in the ``extra_metrics`` parameter.
+        candidate_result: The evaluation result of the candidate model.
+            Returned by the :py:func:`mlflow.evaluate` API.
+        baseline_result: The evaluation result of the baseline model.
+            Returned by the :py:func:`mlflow.evaluate` API.
+            If set to None, the candidate model result will be
+            compared against the threshold values directly.
+
+    Code Example:
+
+        .. code-block:: python
+            :caption: Example of Model Validation
+
+            import mlflow
+            from mlflow.models import MetricThreshold
+
+            thresholds = {
+                "accuracy_score": MetricThreshold(
+                    # accuracy should be >=0.8
+                    threshold=0.8,
+                    # accuracy should be at least 5 percent greater than baseline model accuracy
+                    min_absolute_change=0.05,
+                    # accuracy should be at least 0.05 greater than baseline model accuracy
+                    min_relative_change=0.05,
+                    greater_is_better=True,
+                ),
+            }
+
+            # Get evaluation results for the candidate model
+            candidate_result = mlflow.evaluate(
+                model="<YOUR_CANDIDATE_MODEL_URI>",
+                data=eval_dataset,
+                targets="ground_truth",
+                model_type="classifier",
+            )
+
+            # Get evaluation results for the baseline model
+            baseline_result = mlflow.evaluate(
+                model="<YOUR_BASELINE_MODEL_URI>",
+                data=eval_dataset,
+                targets="ground_truth",
+                model_type="classifier",
+            )
+
+            # Validate the results
+            mlflow.validate_evaluation_results(
+                thresholds,
+                candidate_result,
+                baseline_result,
+            )
+
+        See `the Model Validation documentation
+        <../../models/index.html#performing-model-validation>`_ for more details.
+    """
+    try:
+        assert type(validation_thresholds) is dict
+        for key in validation_thresholds.keys():
+            assert type(key) is str
+        for threshold in validation_thresholds.values():
+            assert isinstance(threshold, MetricThreshold)
+    except AssertionError:
+        raise MlflowException(
+            message="The validation thresholds argument must be a dictionary that maps strings "
+            "to MetricThreshold objects.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+
+    _logger.info("Validating candidate model metrics against baseline")
+    _validate(
+        validation_thresholds,
+        candidate_result.metrics,
+        baseline_result.metrics if baseline_result else {},
+    )
+    _logger.info("Model validation passed!")
+
+
+def _validate(
+    validation_thresholds: dict[str, MetricThreshold],
+    candidate_metrics: dict[str, float],
+    baseline_metrics: dict[str, float],
+):
+    """
+    Validate the model based on validation_thresholds by metrics value and
+    metrics comparison between candidate model's metrics (candidate_metrics) and
+    baseline model's metrics (baseline_metrics).
+
+    Args:
+        validation_thresholds: A dictionary from metric_name to MetricThreshold.
+        candidate_metrics: The metric evaluation result of the candidate model.
+        baseline_metrics: The metric evaluation result of the baseline model.
+
+    Raises:
+        If the validation does not pass, raise an MlflowException with detail failure message.
+    """
+    validation_results = {
+        metric_name: _MetricValidationResult(
+            metric_name,
+            candidate_metrics.get(metric_name),
+            threshold,
+            baseline_metrics.get(metric_name),
+        )
+        for (metric_name, threshold) in validation_thresholds.items()
+    }
+
+    for metric_name, metric_threshold in validation_thresholds.items():
+        validation_result = validation_results[metric_name]
+
+        if metric_name not in candidate_metrics:
+            validation_result.missing_candidate = True
+            continue
+
+        candidate_metric_value = candidate_metrics[metric_name]
+        baseline_metric_value = baseline_metrics[metric_name] if baseline_metrics else None
+
+        # If metric is greater is better, >= is used, otherwise <= is used
+        # for thresholding metric value and model comparison
+        comparator_fn = operator.__ge__ if metric_threshold.greater_is_better else operator.__le__
+        operator_fn = operator.add if metric_threshold.greater_is_better else operator.sub
+
+        if metric_threshold.threshold is not None:
+            # metric threshold fails
+            # - if not (metric_value >= threshold) for greater is better
+            # - if not (metric_value <= threshold) for lower is better
+            validation_result.threshold_failed = not comparator_fn(
+                candidate_metric_value, metric_threshold.threshold
+            )
+
+        if (
+            metric_threshold.min_relative_change or metric_threshold.min_absolute_change
+        ) and metric_name not in baseline_metrics:
+            validation_result.missing_baseline = True
+            continue
+
+        if metric_threshold.min_absolute_change is not None:
+            # metric comparison absolute change fails
+            # - if not (metric_value >= baseline + min_absolute_change) for greater is better
+            # - if not (metric_value <= baseline - min_absolute_change) for lower is better
+            validation_result.min_absolute_change_failed = not comparator_fn(
+                Decimal(candidate_metric_value),
+                Decimal(operator_fn(baseline_metric_value, metric_threshold.min_absolute_change)),
+            )
+
+        if metric_threshold.min_relative_change is not None:
+            # If baseline metric value equals 0, fallback to simple comparison check
+            if baseline_metric_value == 0:
+                _logger.warning(
+                    f"Cannot perform relative model comparison for metric {metric_name} as "
+                    "baseline metric value is 0. Falling back to simple comparison: verifying "
+                    "that candidate metric value is better than the baseline metric value."
+                )
+                validation_result.min_relative_change_failed = not comparator_fn(
+                    Decimal(candidate_metric_value),
+                    Decimal(operator_fn(baseline_metric_value, 1e-10)),
+                )
+                continue
+            # metric comparison relative change fails
+            # - if (metric_value - baseline) / baseline < min_relative_change for greater is better
+            # - if (baseline - metric_value) / baseline < min_relative_change for lower is better
+            if metric_threshold.greater_is_better:
+                relative_change = (
+                    candidate_metric_value - baseline_metric_value
+                ) / baseline_metric_value
+            else:
+                relative_change = (
+                    baseline_metric_value - candidate_metric_value
+                ) / baseline_metric_value
+            validation_result.min_relative_change_failed = (
+                relative_change < metric_threshold.min_relative_change
+            )
+
+    failure_messages = []
+
+    for metric_validation_result in validation_results.values():
+        if metric_validation_result.is_success():
+            continue
+        failure_messages.append(str(metric_validation_result))
+
+    if not failure_messages:
+        return
+
+    raise ModelValidationFailedException(message=os.linesep.join(failure_messages))

mlflow/models/flavor_backend.py

@@ -0,0 +1,93 @@
+from abc import ABCMeta, abstractmethod
+
+from mlflow.utils.annotations import developer_stable
+
+
+@developer_stable
+class FlavorBackend:
+    """
+    Abstract class for Flavor Backend.
+    This class defines the API interface for local model deployment of MLflow model flavors.
+    """
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self, config, **kwargs):
+        self._config = config
+
+    @abstractmethod
+    def predict(self, model_uri, input_path, output_path, content_type):
+        """
+        Generate predictions using a saved MLflow model referenced by the given URI.
+        Input and output are read from and written to a file or stdin / stdout.
+
+        Args:
+            model_uri: URI pointing to the MLflow model to be used for scoring.
+            input_path: Path to the file with input data. If not specified, data is read from
+                        stdin.
+            output_path: Path to the file with output predictions. If not specified, data is
+                         written to stdout.
+            content_type: Specifies the input format. Can be one of {``json``, ``csv``}
+        """
+
+    @abstractmethod
+    def serve(
+        self,
+        model_uri,
+        port,
+        host,
+        timeout,
+        enable_mlserver,
+        synchronous=True,
+        stdout=None,
+        stderr=None,
+    ):
+        """
+        Serve the specified MLflow model locally.
+
+        Args:
+            model_uri: URI pointing to the MLflow model to be used for scoring.
+            port: Port to use for the model deployment.
+            host: Host to use for the model deployment. Defaults to ``localhost``.
+            timeout: Timeout in seconds to serve a request. Defaults to 60.
+            enable_mlserver: Whether to use MLServer or the local scoring server.
+            synchronous: If True, wait until server process exit and return 0, if process exit
+                with non-zero return code, raise exception.
+                If False, return the server process `Popen` instance immediately.
+            stdout: Redirect server stdout
+            stderr: Redirect server stderr
+        """
+
+    def prepare_env(self, model_uri, capture_output=False):
+        """
+        Performs any preparation necessary to predict or serve the model, for example
+        downloading dependencies or initializing a conda environment. After preparation,
+        calling predict or serve should be fast.
+        """
+
+    @abstractmethod
+    def build_image(
+        self, model_uri, image_name, install_mlflow, mlflow_home, enable_mlserver, base_image=None
+    ): ...
+
+    @abstractmethod
+    def generate_dockerfile(
+        self, model_uri, output_path, install_mlflow, mlflow_home, enable_mlserver, base_image=None
+    ): ...
+
+    @abstractmethod
+    def can_score_model(self):
+        """
+        Check whether this flavor backend can be deployed in the current environment.
+
+        Returns:
+            True if this flavor backend can be applied in the current environment.
+        """
+
+    def can_build_image(self):
+        """
+        Returns:
+            True if this flavor has a `build_image` method defined for building a docker
+            container capable of serving the model, False otherwise.
+        """
+        return callable(getattr(self.__class__, "build_image", None))

mlflow/models/flavor_backend_registry.py

@@ -0,0 +1,53 @@
+"""
+Registry of supported flavor backends. Contains a mapping of flavors to flavor backends. This
+mapping is used to select suitable flavor when deploying generic MLflow models.
+
+Flavor backend can deploy particular flavor locally to generate predictions, deploy as a local
+REST api endpoint, or build a docker image for serving the model locally or remotely.
+Not all flavors have a flavor backend.
+"""
+
+import logging
+
+from mlflow.models.model import Model
+from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository
+from mlflow.tracking.artifact_utils import (
+    _get_root_uri_and_artifact_path,
+)
+from mlflow.utils.file_utils import TempDir
+
+_logger = logging.getLogger(__name__)
+
+
+def _get_flavor_backend_for_local_model(model=None, build_docker=True, **kwargs):
+    from mlflow import pyfunc, rfunc
+    from mlflow.pyfunc.backend import PyFuncBackend
+    from mlflow.rfunc.backend import RFuncBackend
+
+    if not model:
+        return pyfunc.FLAVOR_NAME, PyFuncBackend({}, **kwargs)
+
+    backends = {pyfunc.FLAVOR_NAME: PyFuncBackend, rfunc.FLAVOR_NAME: RFuncBackend}
+    for flavor, Backend in backends.items():
+        if flavor in model.flavors:
+            backend = Backend(model.flavors[flavor], **kwargs)
+            if (build_docker and backend.can_build_image()) or backend.can_score_model():
+                return flavor, backend
+
+    return None, None
+
+
+def get_flavor_backend(model_uri, **kwargs):
+    if model_uri:
+        with TempDir() as tmp:
+            root_uri, artifact_path = _get_root_uri_and_artifact_path(model_uri)
+            artifact_repo = get_artifact_repository(root_uri)
+            local_path = artifact_repo.download_artifacts(artifact_path, dst_path=tmp.path())
+            model = Model.load(local_path)
+    else:
+        model = None
+    flavor_name, flavor_backend = _get_flavor_backend_for_local_model(model, **kwargs)
+    if flavor_backend is None:
+        raise Exception("No suitable flavor backend was found for the model.")
+    _logger.info("Selected backend for flavor '%s'", flavor_name)
+    return flavor_backend