genesis-flow 1.0.0__py3-none-any.whl

mlflow/models/evaluation/lift_curve.py

@@ -0,0 +1,178 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def _cumulative_gain_curve(y_true, y_score, pos_label=None):
+    """
+    This method is copied from scikit-plot package.
+    See https://github.com/reiinakano/scikit-plot/blob/2dd3e6a76df77edcbd724c4db25575f70abb57cb/scikitplot/helpers.py#L157
+
+    This function generates the points necessary to plot the Cumulative Gain
+
+    Note: This implementation is restricted to the binary classification task.
+
+    Args:
+        y_true (array-like, shape (n_samples)): True labels of the data.
+
+        y_score (array-like, shape (n_samples)): Target scores, can either be
+            probability estimates of the positive class, confidence values, or
+            non-thresholded measure of decisions (as returned by
+            decision_function on some classifiers).
+
+        pos_label (int or str, default=None): Label considered as positive and
+            others are considered negative
+
+    Returns:
+        percentages (numpy.ndarray): An array containing the X-axis values for
+            plotting the Cumulative Gains chart.
+
+        gains (numpy.ndarray): An array containing the Y-axis values for one
+            curve of the Cumulative Gains chart.
+
+    Raises:
+        ValueError: If `y_true` is not composed of 2 classes. The Cumulative
+            Gain Chart is only relevant in binary classification.
+    """
+    y_true = np.asarray(y_true)
+    y_score = np.asarray(y_score)
+
+    # ensure binary classification if pos_label is not specified
+    classes = np.unique(y_true)
+    if pos_label is None and not (
+        np.array_equal(classes, [0, 1])
+        or np.array_equal(classes, [-1, 1])
+        or np.array_equal(classes, [0])
+        or np.array_equal(classes, [-1])
+        or np.array_equal(classes, [1])
+    ):
+        raise ValueError("Data is not binary and pos_label is not specified")
+    elif pos_label is None:
+        pos_label = 1.0
+
+    # make y_true a boolean vector
+    y_true = y_true == pos_label
+
+    sorted_indices = np.argsort(y_score)[::-1]
+    y_true = y_true[sorted_indices]
+    gains = np.cumsum(y_true)
+
+    percentages = np.arange(start=1, stop=len(y_true) + 1)
+
+    gains = gains / float(np.sum(y_true))
+    percentages = percentages / float(len(y_true))
+
+    gains = np.insert(gains, 0, [0])
+    percentages = np.insert(percentages, 0, [0])
+
+    return percentages, gains
+
+
+def plot_lift_curve(
+    y_true,
+    y_probas,
+    title="Lift Curve",
+    ax=None,
+    figsize=None,
+    title_fontsize="large",
+    text_fontsize="medium",
+    pos_label=None,
+):
+    """
+    This method is copied from scikit-plot package.
+    See https://github.com/reiinakano/scikit-plot/blob/2dd3e6a76df77edcbd724c4db25575f70abb57cb/scikitplot/metrics.py#L1133
+
+    Generates the Lift Curve from labels and scores/probabilities
+
+    The lift curve is used to determine the effectiveness of a
+    binary classifier. A detailed explanation can be found at
+    http://www2.cs.uregina.ca/~dbd/cs831/notes/lift_chart/lift_chart.html.
+    The implementation here works only for binary classification.
+
+    Args:
+        y_true (array-like, shape (n_samples)):
+            Ground truth (correct) target values.
+
+        y_probas (array-like, shape (n_samples, n_classes)):
+            Prediction probabilities for each class returned by a classifier.
+
+        title (string, optional): Title of the generated plot. Defaults to
+            "Lift Curve".
+
+        ax (:class:`matplotlib.axes.Axes`, optional): The axes upon which to
+            plot the learning curve. If None, the plot is drawn on a new set of
+            axes.
+
+        figsize (2-tuple, optional): Tuple denoting figure size of the plot
+            e.g. (6, 6). Defaults to ``None``.
+
+        title_fontsize (string or int, optional): Matplotlib-style fontsizes.
+            Use e.g. "small", "medium", "large" or integer-values. Defaults to
+            "large".
+
+        text_fontsize (string or int, optional): Matplotlib-style fontsizes.
+            Use e.g. "small", "medium", "large" or integer-values. Defaults to
+            "medium".
+
+        pos_label (optional): Label for the positive class.
+
+    Returns:
+        ax (:class:`matplotlib.axes.Axes`): The axes on which the plot was
+            drawn.
+
+    Example:
+        >>> lr = LogisticRegression()
+        >>> lr = lr.fit(X_train, y_train)
+        >>> y_probas = lr.predict_proba(X_test)
+        >>> plot_lift_curve(y_test, y_probas)
+        <matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
+        >>> plt.show()
+
+        .. image:: _static/examples/plot_lift_curve.png
+           :align: center
+           :alt: Lift Curve
+    """
+    y_true = np.array(y_true)
+    y_probas = np.array(y_probas)
+
+    classes = np.unique(y_true)
+    if len(classes) != 2:
+        raise ValueError(f"Cannot calculate Lift Curve for data with {len(classes)} category/ies")
+
+    # Compute Cumulative Gain Curves
+    percentages, gains1 = _cumulative_gain_curve(y_true, y_probas[:, 0], classes[0])
+    percentages, gains2 = _cumulative_gain_curve(y_true, y_probas[:, 1], classes[1])
+
+    percentages = percentages[1:]
+    gains1 = gains1[1:]
+    gains2 = gains2[1:]
+
+    gains1 = gains1 / percentages
+    gains2 = gains2 / percentages
+
+    if ax is None:
+        _, ax = plt.subplots(1, 1, figsize=figsize)
+
+    ax.set_title(title, fontsize=title_fontsize)
+
+    label0 = f"Class {classes[0]}"
+    label1 = f"Class {classes[1]}"
+    # show (positive) next to the positive class in the legend
+    if pos_label:
+        if pos_label == classes[0]:
+            label0 = f"Class {classes[0]} (positive)"
+        elif pos_label == classes[1]:
+            label1 = f"Class {classes[1]} (positive)"
+        # do not mark positive class if pos_label is not in classes
+
+    ax.plot(percentages, gains1, lw=3, label=label0)
+    ax.plot(percentages, gains2, lw=3, label=label1)
+
+    ax.plot([0, 1], [1, 1], "k--", lw=2, label="Baseline")
+
+    ax.set_xlabel("Percentage of sample", fontsize=text_fontsize)
+    ax.set_ylabel("Lift", fontsize=text_fontsize)
+    ax.tick_params(labelsize=text_fontsize)
+    ax.grid("on")
+    ax.legend(loc="best", fontsize=text_fontsize)
+
+    return ax

mlflow/models/evaluation/utils/metric.py

@@ -0,0 +1,123 @@
+import logging
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+import numpy as np
+
+from mlflow.metrics.base import MetricValue
+from mlflow.models.evaluation.base import EvaluationMetric
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MetricDefinition:
+    """
+    A namedtuple representing a metric function and its properties.
+
+    function : the metric function
+    name : the name of the metric function
+    index : the index of the function in the ``extra_metrics`` argument of mlflow.evaluate
+    """
+
+    function: Callable[..., Any]
+    name: str
+    index: int
+    version: Optional[str] = None
+    genai_metric_args: Optional[dict[str, Any]] = None
+
+    @classmethod
+    def from_index_and_metric(cls, index: int, metric: EvaluationMetric):
+        return cls(
+            function=metric.eval_fn,
+            index=index,
+            name=metric.name,
+            version=metric.version,
+            genai_metric_args=metric.genai_metric_args,
+        )
+
+    def evaluate(self, eval_fn_args) -> Optional[MetricValue]:
+        """
+        This function calls the metric function and performs validations on the returned
+        result to ensure that they are in the expected format. It will warn and will not log metrics
+        that are in the wrong format.
+
+        Args:
+            eval_fn_args: A dictionary of args needed to compute the eval metrics.
+
+        Returns:
+            MetricValue
+        """
+        if self.index < 0:
+            exception_header = f"Did not log builtin metric '{self.name}' because it"
+        else:
+            exception_header = (
+                f"Did not log metric '{self.name}' at index "
+                f"{self.index} in the `extra_metrics` parameter because it"
+            )
+
+        metric: MetricValue = self.function(*eval_fn_args)
+
+        def _is_numeric(value):
+            return isinstance(value, (int, float, np.number))
+
+        def _is_string(value):
+            return isinstance(value, str)
+
+        if metric is None:
+            _logger.warning(f"{exception_header} returned None.")
+            return
+
+        if _is_numeric(metric):
+            return MetricValue(aggregate_results={self.name: metric})
+
+        if not isinstance(metric, MetricValue):
+            _logger.warning(f"{exception_header} did not return a MetricValue.")
+            return
+
+        scores = metric.scores
+        justifications = metric.justifications
+        aggregates = metric.aggregate_results
+
+        if scores is not None:
+            if not isinstance(scores, list):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with scores as a list."
+                )
+                return
+            if any(not (_is_numeric(s) or _is_string(s) or s is None) for s in scores):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with numeric or string scores."
+                )
+                return
+
+        if justifications is not None:
+            if not isinstance(justifications, list):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with justifications as a list."
+                )
+                return
+            if any(not (_is_string(just) or just is None) for just in justifications):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with string justifications."
+                )
+                return
+
+        if aggregates is not None:
+            if not isinstance(aggregates, dict):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with aggregate_results as a dict."
+                )
+                return
+
+            if any(
+                not (isinstance(k, str) and (_is_numeric(v) or v is None))
+                for k, v in aggregates.items()
+            ):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with aggregate_results with "
+                    "str keys and numeric values."
+                )
+                return
+
+        return metric

mlflow/models/evaluation/utils/trace.py

@@ -0,0 +1,179 @@
+import contextlib
+import inspect
+import logging
+from typing import Any, Callable
+
+from mlflow.ml_package_versions import FLAVOR_TO_MODULE_NAME
+from mlflow.utils.autologging_utils import (
+    AUTOLOGGING_INTEGRATIONS,
+    autologging_conf_lock,
+    get_autolog_function,
+    is_autolog_supported,
+)
+from mlflow.utils.autologging_utils.safety import revert_patches
+from mlflow.utils.import_hooks import (
+    _post_import_hooks,
+    get_post_import_hooks,
+    register_post_import_hook,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+# This flag is used to display the message only once when tracing is enabled during the evaluation.
+_SHOWN_TRACE_MESSAGE_BEFORE = False
+
+
+@contextlib.contextmanager
+@autologging_conf_lock
+def configure_autologging_for_evaluation(enable_tracing: bool = True):
+    """
+    Temporarily override the autologging configuration for all flavors during the model evaluation.
+    For example, model auto-logging must be disabled during the evaluation. After the evaluation
+    is done, the original autologging configurations are restored.
+
+    Args:
+        enable_tracing (bool): Whether to enable tracing for the supported flavors during eval.
+    """
+    original_import_hooks = {}
+    new_import_hooks = {}
+
+    # AUTOLOGGING_INTEGRATIONS can change during we iterate over flavors and enable/disable
+    # autologging, therefore, we snapshot the current configuration to restore it later.
+    global_config_snapshot = AUTOLOGGING_INTEGRATIONS.copy()
+
+    for flavor in FLAVOR_TO_MODULE_NAME:
+        if not is_autolog_supported(flavor):
+            continue
+
+        original_config = global_config_snapshot.get(flavor, {}).copy()
+
+        # If autologging is explicitly disabled, do nothing.
+        if original_config.get("disable", False):
+            continue
+
+        # NB: Using post-import hook to configure the autologging lazily when the target
+        # flavor's module is imported, rather than configuring it immediately. This is
+        # because the evaluation code usually only uses a subset of the supported flavors,
+        # hence we want to avoid unnecessary overhead of configuring all flavors.
+        @autologging_conf_lock
+        def _setup_autolog(module):
+            try:
+                autolog = get_autolog_function(flavor)
+
+                # If tracing is supported and not explicitly disabled, enable it.
+                if enable_tracing and _should_enable_tracing(flavor, global_config_snapshot):
+                    new_config = {
+                        k: False if k.startswith("log_") else v for k, v in original_config.items()
+                    }
+                    # log_models needs to be disabled
+                    # so we don't init LoggedModels during eval for some GenAI flavors
+                    new_config |= {"log_traces": True, "silent": True, "log_models": False}
+                    _kwargs_safe_invoke(autolog, new_config)
+
+                    global _SHOWN_TRACE_MESSAGE_BEFORE
+                    if not _SHOWN_TRACE_MESSAGE_BEFORE:
+                        _logger.info(
+                            "Auto tracing is temporarily enabled during the model evaluation "
+                            "for computing some metrics and debugging. To disable tracing, call "
+                            "`mlflow.autolog(disable=True)`."
+                        )
+                        _SHOWN_TRACE_MESSAGE_BEFORE = True
+                else:
+                    autolog(disable=True)
+
+            except Exception:
+                _logger.debug(f"Failed to update autologging config for {flavor}.", exc_info=True)
+
+        module = FLAVOR_TO_MODULE_NAME[flavor]
+        try:
+            original_import_hooks[module] = get_post_import_hooks(module)
+            new_import_hooks[module] = _setup_autolog
+            register_post_import_hook(_setup_autolog, module, overwrite=True)
+        except Exception:
+            _logger.debug(f"Failed to register post-import hook for {flavor}.", exc_info=True)
+
+    try:
+        yield
+    finally:
+        # Remove post-import hooks and patches the are registered during the evaluation.
+        for module, hooks in new_import_hooks.items():
+            # Restore original post-import hooks if any. Note that we don't use
+            # register_post_import_hook method to bypass some pre-checks and just
+            # restore the original state.
+            if hooks is None:
+                _post_import_hooks.pop(module, None)
+            else:
+                _post_import_hooks[module] = original_import_hooks[module]
+
+        # If any autologging configuration is updated, restore original autologging configurations.
+        for flavor, new_config in AUTOLOGGING_INTEGRATIONS.copy().items():
+            original_config = global_config_snapshot.get(flavor)
+            if original_config != new_config:
+                try:
+                    autolog = get_autolog_function(flavor)
+                    if original_config:
+                        _kwargs_safe_invoke(autolog, original_config)
+                        AUTOLOGGING_INTEGRATIONS[flavor] = original_config
+                    else:
+                        # If the original configuration is empty, autologging was not enabled before
+                        autolog(disable=True)
+                        # Remove all safe_patch applied by autologging
+                        revert_patches(flavor)
+                        # We also need to remove the config entry from AUTOLOGGING_INTEGRATIONS,
+                        # so as not to confuse with the case user explicitly disabled autologging.
+                        AUTOLOGGING_INTEGRATIONS.pop(flavor, None)
+                except ImportError:
+                    pass
+                except Exception as e:
+                    if original_config is None or (
+                        not original_config.get("disable", False)
+                        and not original_config.get("silent", False)
+                    ):
+                        _logger.warning(
+                            f"Exception raised while calling autologging for {flavor}: {e}"
+                        )
+
+
+def _should_enable_tracing(flavor: str, autologging_config: dict[str, Any]) -> bool:
+    """
+    Check if tracing should be enabled for the given flavor during the model evaluation.
+    """
+    # 1. Check if the autologging or tracing is globally disabled
+    # TODO: This check should not take precedence over the flavor-specific configuration
+    # set by the explicit mlflow.<flavor>.autolog() call by users.
+    # However, in Databricks, sometimes mlflow.<flavor>.autolog() is automatically
+    # called in the kernel startup, which is confused with the user's action. In
+    # such cases, even when user disables autologging globally, the flavor-specific
+    # autologging remains enabled. We are going to fix the Databricks side issue,
+    # and after that, we should move this check down after the flavor-specific check.
+    global_config = autologging_config.get("mlflow", {})
+    if global_config.get("disable", False) or (not global_config.get("log_traces", True)):
+        return False
+
+    if not _is_trace_autologging_supported(flavor):
+        return False
+
+    # 3. Check if tracing is explicitly disabled for the flavor
+    flavor_config = autologging_config.get(flavor, {})
+    return flavor_config.get("log_traces", True)
+
+
+def _kwargs_safe_invoke(func: Callable[..., Any], kwargs: dict[str, Any]):
+    """
+    Invoke the function with the given dictionary as keyword arguments, but only include the
+    arguments that are present in the function's signature.
+
+    This is particularly used for calling autolog() function with the configuration dictionary
+    stored in AUTOLOGGING_INTEGRATIONS. While the config keys mostly align with the autolog()'s
+    signature by design, some keys are not present in autolog(), such as "globally_configured".
+    """
+    sig = inspect.signature(func)
+    return func(**{k: v for k, v in kwargs.items() if k in sig.parameters})
+
+
+def _is_trace_autologging_supported(flavor_name: str) -> bool:
+    """Check if the given flavor supports trace autologging."""
+    if autolog_func := get_autolog_function(flavor_name):
+        return "log_traces" in inspect.signature(autolog_func).parameters
+    return False