genesis-flow 1.0.0__py3-none-any.whl

mlflow/models/evaluation/evaluators/default.py

@@ -0,0 +1,233 @@
+import logging
+import os
+import time
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+import mlflow
+from mlflow.entities.metric import Metric
+from mlflow.exceptions import MlflowException
+from mlflow.metrics import (
+    MetricValue,
+    ari_grade_level,
+    exact_match,
+    flesch_kincaid_grade_level,
+    ndcg_at_k,
+    precision_at_k,
+    recall_at_k,
+    rouge1,
+    rouge2,
+    rougeL,
+    rougeLsum,
+    token_count,
+    toxicity,
+)
+from mlflow.metrics.genai.genai_metric import _GENAI_CUSTOM_METRICS_FILE_NAME
+from mlflow.models.evaluation.artifacts import JsonEvaluationArtifact
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    _LATENCY_METRIC_NAME,
+    BuiltInEvaluator,
+    _extract_output_and_other_columns,
+    _extract_predict_fn,
+)
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+
+_logger = logging.getLogger(__name__)
+
+
+class DefaultEvaluator(BuiltInEvaluator):
+    """
+    The default built-in evaluator for any models that cannot be evaluated
+    by other built-in evaluators, such as question-answering.
+    """
+
+    name = "default"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        return model_type in _ModelType.values() or model_type is None
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        compute_latency = False
+        for extra_metric in extra_metrics:
+            # If latency metric is specified, we will compute latency for the model
+            # during prediction, and we will remove the metric from the list of extra
+            # metrics to be computed after prediction.
+            if extra_metric.name == _LATENCY_METRIC_NAME:
+                compute_latency = True
+                extra_metrics.remove(extra_metric)
+        self._log_genai_custom_metrics(extra_metrics)
+
+        # Generate model predictions and evaluate metrics
+        y_pred, other_model_outputs, self.predictions = self._generate_model_predictions(
+            model, input_df=self.X.copy_to_avoid_mutation(), compute_latency=compute_latency
+        )
+        y_true = self.dataset.labels_data
+
+        metrics = self._builtin_metrics() + extra_metrics
+        self.evaluate_metrics(
+            metrics,
+            prediction=y_pred,
+            target=self.dataset.labels_data,
+            other_output_df=other_model_outputs,
+        )
+        self.evaluate_and_log_custom_artifacts(custom_artifacts, prediction=y_pred, target=y_true)
+
+        # Log metrics and artifacts
+        self.log_metrics()
+        self.log_eval_table(y_pred, other_model_outputs)
+        return EvaluationResult(
+            metrics=self.aggregate_metrics, artifacts=self.artifacts, run_id=self.run_id
+        )
+
+    def _builtin_metrics(self) -> list[Metric]:
+        """
+        Get a list of builtin metrics for the model type.
+        """
+        text_metrics = [
+            token_count(),
+            toxicity(),
+            flesch_kincaid_grade_level(),
+            ari_grade_level(),
+        ]
+        builtin_metrics = []
+
+        # NB: Classifier and Regressor are handled by dedicated built-in evaluators,
+        if self.model_type == _ModelType.QUESTION_ANSWERING:
+            builtin_metrics = [*text_metrics, exact_match()]
+        elif self.model_type == _ModelType.TEXT_SUMMARIZATION:
+            builtin_metrics = [
+                *text_metrics,
+                rouge1(),
+                rouge2(),
+                rougeL(),
+                rougeLsum(),
+            ]
+        elif self.model_type == _ModelType.TEXT:
+            builtin_metrics = text_metrics
+        elif self.model_type == _ModelType.RETRIEVER:
+            # default k to 3 if not specified
+            retriever_k = self.evaluator_config.pop("retriever_k", 3)
+            builtin_metrics = [
+                precision_at_k(retriever_k),
+                recall_at_k(retriever_k),
+                ndcg_at_k(retriever_k),
+            ]
+
+        return builtin_metrics
+
+    def _generate_model_predictions(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        input_df: pd.DataFrame,
+        compute_latency=False,
+    ):
+        """
+        Helper method for generating model predictions
+        """
+        predict_fn = _extract_predict_fn(model)
+
+        def predict_with_latency(X_copy):
+            y_pred_list = []
+            pred_latencies = []
+            if len(X_copy) == 0:
+                raise ValueError("Empty input data")
+
+            is_dataframe = isinstance(X_copy, pd.DataFrame)
+
+            for row in X_copy.iterrows() if is_dataframe else enumerate(X_copy):
+                i, row_data = row
+                single_input = row_data.to_frame().T if is_dataframe else row_data
+                start_time = time.time()
+                y_pred = predict_fn(single_input)
+                end_time = time.time()
+                pred_latencies.append(end_time - start_time)
+                y_pred_list.append(y_pred)
+
+            # Update latency metric
+            self.metrics_values.update({_LATENCY_METRIC_NAME: MetricValue(scores=pred_latencies)})
+
+            # Aggregate all predictions into model_predictions
+            sample_pred = y_pred_list[0]
+            if isinstance(sample_pred, pd.DataFrame):
+                return pd.concat(y_pred_list)
+            elif isinstance(sample_pred, np.ndarray):
+                return np.concatenate(y_pred_list, axis=0)
+            elif isinstance(sample_pred, list):
+                return sum(y_pred_list, [])
+            elif isinstance(sample_pred, pd.Series):
+                return pd.concat(y_pred_list, ignore_index=True)
+            elif isinstance(sample_pred, str):
+                return y_pred_list
+            else:
+                raise MlflowException(
+                    message=f"Unsupported prediction type {type(sample_pred)} for model type "
+                    f"{self.model_type}.",
+                    error_code=INVALID_PARAMETER_VALUE,
+                )
+
+        if model is not None:
+            _logger.info("Computing model predictions.")
+
+            if compute_latency:
+                model_predictions = predict_with_latency(input_df)
+            else:
+                model_predictions = predict_fn(input_df)
+        else:
+            if compute_latency:
+                _logger.warning(
+                    "Setting the latency to 0 for all entries because the model is not provided."
+                )
+                self.metrics_values.update(
+                    {_LATENCY_METRIC_NAME: MetricValue(scores=[0.0] * len(input_df))}
+                )
+            model_predictions = self.dataset.predictions_data
+
+        output_column_name = self.predictions
+        (
+            y_pred,
+            other_output_df,
+            predictions_column_name,
+        ) = _extract_output_and_other_columns(model_predictions, output_column_name)
+
+        return y_pred, other_output_df, predictions_column_name
+
+    def _log_genai_custom_metrics(self, extra_metrics: list[EvaluationMetric]):
+        genai_custom_metrics = [
+            extra_metric.genai_metric_args
+            for extra_metric in extra_metrics
+            # When the field is present, the metric is created from either make_genai_metric
+            # or make_genai_metric_from_prompt. We will log the metric definition.
+            if extra_metric.genai_metric_args is not None
+        ]
+
+        if len(genai_custom_metrics) == 0:
+            return
+
+        names = []
+        versions = []
+        metric_args_list = []
+
+        for metric_args in genai_custom_metrics:
+            names.append(metric_args["name"])
+            # Custom metrics created from make_genai_metric_from_prompt don't have version
+            versions.append(metric_args.get("version", ""))
+            metric_args_list.append(metric_args)
+
+        data = {"name": names, "version": versions, "metric_args": metric_args_list}
+
+        mlflow.log_table(data, artifact_file=_GENAI_CUSTOM_METRICS_FILE_NAME)
+
+        artifact_name = os.path.splitext(_GENAI_CUSTOM_METRICS_FILE_NAME)[0]
+        self.artifacts[artifact_name] = JsonEvaluationArtifact(
+            uri=mlflow.get_artifact_uri(_GENAI_CUSTOM_METRICS_FILE_NAME)
+        )

mlflow/models/evaluation/evaluators/regressor.py

@@ -0,0 +1,96 @@
+from typing import Optional
+
+import numpy as np
+from sklearn import metrics as sk_metrics
+
+import mlflow
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    BuiltInEvaluator,
+    _extract_output_and_other_columns,
+    _extract_predict_fn,
+    _get_aggregate_metrics_values,
+)
+
+
+class RegressorEvaluator(BuiltInEvaluator):
+    """
+    A built-in evaluator for regressor models.
+    """
+
+    name = "regressor"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        return model_type == _ModelType.REGRESSOR
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        self.y_true = self.dataset.labels_data
+        self.sample_weights = self.evaluator_config.get("sample_weights", None)
+
+        input_df = self.X.copy_to_avoid_mutation()
+        self.y_pred = self._generate_model_predictions(model, input_df)
+        self._compute_buildin_metrics(model)
+
+        self.evaluate_metrics(extra_metrics, prediction=self.y_pred, target=self.y_true)
+        self.evaluate_and_log_custom_artifacts(
+            custom_artifacts, prediction=self.y_pred, target=self.y_true
+        )
+
+        self.log_metrics()
+        self.log_eval_table(self.y_pred)
+
+        return EvaluationResult(
+            metrics=self.aggregate_metrics, artifacts=self.artifacts, run_id=self.run_id
+        )
+
+    def _generate_model_predictions(self, model, input_df):
+        if predict_fn := _extract_predict_fn(model):
+            preds = predict_fn(input_df)
+            y_pred, _, _ = _extract_output_and_other_columns(preds, self.predictions)
+            return y_pred
+        else:
+            return self.dataset.predictions_data
+
+    def _compute_buildin_metrics(self, model):
+        self._evaluate_sklearn_model_score_if_scorable(model, self.y_true, self.sample_weights)
+        self.metrics_values.update(
+            _get_aggregate_metrics_values(
+                _get_regressor_metrics(self.y_true, self.y_pred, self.sample_weights)
+            )
+        )
+
+
+def _get_regressor_metrics(y, y_pred, sample_weights):
+    from mlflow.metrics.metric_definitions import _root_mean_squared_error
+
+    sum_on_target = (
+        (np.array(y) * np.array(sample_weights)).sum() if sample_weights is not None else sum(y)
+    )
+    return {
+        "example_count": len(y),
+        "mean_absolute_error": sk_metrics.mean_absolute_error(
+            y, y_pred, sample_weight=sample_weights
+        ),
+        "mean_squared_error": sk_metrics.mean_squared_error(
+            y, y_pred, sample_weight=sample_weights
+        ),
+        "root_mean_squared_error": _root_mean_squared_error(
+            y_true=y,
+            y_pred=y_pred,
+            sample_weight=sample_weights,
+        ),
+        "sum_on_target": sum_on_target,
+        "mean_on_target": sum_on_target / len(y),
+        "r2_score": sk_metrics.r2_score(y, y_pred, sample_weight=sample_weights),
+        "max_error": sk_metrics.max_error(y, y_pred),
+        "mean_absolute_percentage_error": sk_metrics.mean_absolute_percentage_error(
+            y, y_pred, sample_weight=sample_weights
+        ),
+    }

mlflow/models/evaluation/evaluators/shap.py

@@ -0,0 +1,296 @@
+import functools
+import logging
+from typing import Optional
+
+import numpy as np
+from packaging.version import Version
+from sklearn.pipeline import Pipeline as sk_Pipeline
+
+import mlflow
+from mlflow import MlflowException
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    BuiltInEvaluator,
+    _extract_predict_fn,
+    _extract_raw_model,
+    _get_dataframe_with_renamed_columns,
+)
+from mlflow.models.evaluation.evaluators.classifier import (
+    _is_continuous,
+    _suppress_class_imbalance_errors,
+)
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+from mlflow.pyfunc import _ServedPyFuncModel
+
+_logger = logging.getLogger(__name__)
+
+
+_SUPPORTED_SHAP_ALGORITHMS = ("exact", "permutation", "partition", "kernel")
+_DEFAULT_SAMPLE_ROWS_FOR_SHAP = 2000
+
+
+def _shap_predict_fn(x, predict_fn, feature_names):
+    return predict_fn(_get_dataframe_with_renamed_columns(x, feature_names))
+
+
+class ShapEvaluator(BuiltInEvaluator):
+    """
+    A built-in evaluator to get SHAP explainability insights for classifier and regressor models.
+
+    This evaluator often run with the main evaluator for the model like ClassifierEvaluator.
+    """
+
+    name = "shap"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        return model_type in (_ModelType.CLASSIFIER, _ModelType.REGRESSOR) and evaluator_config.get(
+            "log_model_explainability", True
+        )
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        if isinstance(model, _ServedPyFuncModel):
+            _logger.warning(
+                "Skipping model explainability because a model server is used for environment "
+                "restoration."
+            )
+            return
+
+        model_loader_module, raw_model = _extract_raw_model(model)
+        if model_loader_module == "mlflow.spark":
+            # TODO: Shap explainer need to manipulate on each feature values,
+            #  but spark model input dataframe contains Vector type feature column
+            #  which shap explainer does not support.
+            #  To support this, we need expand the Vector type feature column into
+            #  multiple scalar feature columns and pass it to shap explainer.
+            _logger.warning(
+                "Logging model explainability insights is not currently supported for PySpark "
+                "models."
+            )
+            return
+
+        self.y_true = self.dataset.labels_data
+        self.label_list = self.evaluator_config.get("label_list")
+        self.pos_label = self.evaluator_config.get("pos_label")
+
+        if not (np.issubdtype(self.y_true.dtype, np.number) or self.y_true.dtype == np.bool_):
+            # Note: python bool type inherits number type but np.bool_ does not inherit np.number.
+            _logger.warning(
+                "Skip logging model explainability insights because it requires all label "
+                "values to be numeric or boolean."
+            )
+            return
+
+        algorithm = self.evaluator_config.get("explainability_algorithm", None)
+        if algorithm is not None and algorithm not in _SUPPORTED_SHAP_ALGORITHMS:
+            raise MlflowException(
+                message=f"Specified explainer algorithm {algorithm} is unsupported. Currently only "
+                f"support {','.join(_SUPPORTED_SHAP_ALGORITHMS)} algorithms.",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+
+        if algorithm != "kernel":
+            feature_dtypes = list(self.X.get_original().dtypes)
+            for feature_dtype in feature_dtypes:
+                if not np.issubdtype(feature_dtype, np.number):
+                    _logger.warning(
+                        "Skip logging model explainability insights because the shap explainer "
+                        f"{algorithm} requires all feature values to be numeric, and each feature "
+                        "column must only contain scalar values."
+                    )
+                    return
+
+        try:
+            import shap
+            from matplotlib import pyplot
+        except ImportError:
+            _logger.warning(
+                "SHAP or matplotlib package is not installed, so model explainability insights "
+                "will not be logged."
+            )
+            return
+
+        if Version(shap.__version__) < Version("0.40"):
+            _logger.warning(
+                "Shap package version is lower than 0.40, Skip log model explainability."
+            )
+            return
+
+        sample_rows = self.evaluator_config.get(
+            "explainability_nsamples", _DEFAULT_SAMPLE_ROWS_FOR_SHAP
+        )
+
+        X_df = self.X.copy_to_avoid_mutation()
+
+        sampled_X = shap.sample(X_df, sample_rows, random_state=0)
+
+        mode_or_mean_dict = _compute_df_mode_or_mean(X_df)
+        sampled_X = sampled_X.fillna(mode_or_mean_dict)
+
+        # shap explainer might call provided `predict_fn` with a `numpy.ndarray` type
+        # argument, this might break some model inference, so convert the argument into
+        # a pandas dataframe.
+        # The `shap_predict_fn` calls model's predict function, we need to restore the input
+        # dataframe with original column names, because some model prediction routine uses
+        # the column name.
+
+        predict_fn = _extract_predict_fn(model)
+        shap_predict_fn = functools.partial(
+            _shap_predict_fn, predict_fn=predict_fn, feature_names=self.dataset.feature_names
+        )
+
+        if self.label_list is None:
+            # If label list is not specified, infer label list from model output.
+            # We need to copy the input data as the model might mutate the input data.
+            y_pred = predict_fn(X_df.copy()) if predict_fn else self.dataset.predictions_data
+            self.label_list = np.unique(np.concatenate([self.y_true, y_pred]))
+
+        try:
+            if algorithm:
+                if algorithm == "kernel":
+                    # We need to lazily import shap, so lazily import `_PatchedKernelExplainer`
+                    from mlflow.models.evaluation._shap_patch import _PatchedKernelExplainer
+
+                    kernel_link = self.evaluator_config.get(
+                        "explainability_kernel_link", "identity"
+                    )
+                    if kernel_link not in ["identity", "logit"]:
+                        raise ValueError(
+                            "explainability_kernel_link config can only be set to 'identity' or "
+                            f"'logit', but got '{kernel_link}'."
+                        )
+                    background_X = shap.sample(X_df, sample_rows, random_state=3)
+                    background_X = background_X.fillna(mode_or_mean_dict)
+
+                    explainer = _PatchedKernelExplainer(
+                        shap_predict_fn, background_X, link=kernel_link
+                    )
+                else:
+                    explainer = shap.Explainer(
+                        shap_predict_fn,
+                        sampled_X,
+                        feature_names=self.dataset.feature_names,
+                        algorithm=algorithm,
+                    )
+            else:
+                if (
+                    raw_model
+                    and not len(self.label_list) > 2
+                    and not isinstance(raw_model, sk_Pipeline)
+                ):
+                    # For mulitnomial classifier, shap.Explainer may choose Tree/Linear explainer
+                    # for raw model, this case shap plot doesn't support it well, so exclude the
+                    # multinomial_classifier case here.
+                    explainer = shap.Explainer(
+                        raw_model, sampled_X, feature_names=self.dataset.feature_names
+                    )
+                else:
+                    # fallback to default explainer
+                    explainer = shap.Explainer(
+                        shap_predict_fn, sampled_X, feature_names=self.dataset.feature_names
+                    )
+
+            _logger.info(f"Shap explainer {explainer.__class__.__name__} is used.")
+
+            if algorithm == "kernel":
+                shap_values = shap.Explanation(
+                    explainer.shap_values(sampled_X), feature_names=self.dataset.feature_names
+                )
+            else:
+                shap_values = explainer(sampled_X)
+        except Exception as e:
+            # Shap evaluation might fail on some edge cases, e.g., unsupported input data values
+            # or unsupported model on specific shap explainer. Catch exception to prevent it
+            # breaking the whole `evaluate` function.
+
+            if not self.evaluator_config.get("ignore_exceptions", True):
+                raise e
+
+            _logger.warning(
+                f"Shap evaluation failed. Reason: {e!r}. "
+                "Set logging level to DEBUG to see the full traceback."
+            )
+            _logger.debug("", exc_info=True)
+            return
+
+        if self.evaluator_config.get("log_explainer", False):
+            try:
+                mlflow.shap.log_explainer(explainer, name="explainer")
+            except Exception as e:
+                # TODO: The explainer saver is buggy, if `get_underlying_model_flavor` return
+                #   "unknown", then fallback to shap explainer saver, and shap explainer will call
+                #   `model.save` for sklearn model, there is no `.save` method, so error will
+                #   happen.
+                _logger.warning(
+                    f"Logging explainer failed. Reason: {e!r}. "
+                    "Set logging level to DEBUG to see the full traceback."
+                )
+                _logger.debug("", exc_info=True)
+
+        def _adjust_color_bar():
+            pyplot.gcf().axes[-1].set_aspect("auto")
+            pyplot.gcf().axes[-1].set_box_aspect(50)
+
+        def _adjust_axis_tick():
+            pyplot.xticks(fontsize=10)
+            pyplot.yticks(fontsize=10)
+
+        def plot_beeswarm():
+            shap.plots.beeswarm(shap_values, show=False, color_bar=True)
+            _adjust_color_bar()
+            _adjust_axis_tick()
+
+        with _suppress_class_imbalance_errors(ValueError, log_warning=False):
+            self._log_image_artifact(
+                plot_beeswarm,
+                "shap_beeswarm_plot",
+            )
+
+        def plot_summary():
+            shap.summary_plot(shap_values, show=False, color_bar=True)
+            _adjust_color_bar()
+            _adjust_axis_tick()
+
+        with _suppress_class_imbalance_errors(TypeError, log_warning=False):
+            self._log_image_artifact(
+                plot_summary,
+                "shap_summary_plot",
+            )
+
+        def plot_feature_importance():
+            shap.plots.bar(shap_values, show=False)
+            _adjust_axis_tick()
+
+        with _suppress_class_imbalance_errors(IndexError, log_warning=False):
+            self._log_image_artifact(
+                plot_feature_importance,
+                "shap_feature_importance_plot",
+            )
+
+        return EvaluationResult(
+            metrics=self.aggregate_metrics,
+            artifacts=self.artifacts,
+            run_id=self.run_id,
+        )
+
+
+def _compute_df_mode_or_mean(df):
+    """
+    Compute mean (for continuous columns) and compute mode (for other columns) for the
+    input dataframe, return a dict, key is column name, value is the corresponding mode or
+    mean value, this function calls `_is_continuous` to determine whether the
+    column is continuous column.
+    """
+    continuous_cols = [c for c in df.columns if _is_continuous(df[c])]
+    df_cont = df[continuous_cols]
+    df_non_cont = df.drop(continuous_cols, axis=1)
+
+    means = {} if df_cont.empty else df_cont.mean().to_dict()
+    modes = {} if df_non_cont.empty else df_non_cont.mode().loc[0].to_dict()
+    return {**means, **modes}