oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/evaluations/evaluation_plot.py

@@ -1,983 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from __future__ import print_function, absolute_import, division
-
-import base64
-from io import BytesIO
-import matplotlib as mpl
-import matplotlib.pyplot as plt
-import matplotlib.lines as mlines
-from matplotlib.ticker import FormatStrFormatter
-import numpy as np
-import math
-from ads.common import logger
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
-import itertools
-import pandas as pd
-
-MAX_TITLE_LEN = 20
-MAX_LEGEND_LEN = 10
-MAX_PLOTS_PER_ROW = 2
-# Maximum class number evaluation plotting supporting for multiclass problems
-MAX_PLOTTING_CLASSES = 10
-# Maximum characters in class label able to be shown without being truncated
-MAX_CHARACTERS_LEN = 13
-
-
-def _fig_to_html(fig):
-    tmpfile = BytesIO()
-    fig.savefig(tmpfile, format="png")
-    encoded = base64.b64encode(tmpfile.getvalue()).decode("utf-8")
-
-    html = "<img src='data:image/png;base64,{}'>".format(encoded)
-    return html
-
-
-class EvaluationPlot:
-    """EvaluationPlot holds data and methods for plots and it used to output them
-
-    Attributes
-    ----------
-    baseline (bool):
-        whether to plot the null model or zero information model
-    baseline_kwargs (dict):
-        keyword arguments for the baseline plot
-    color_wheel (dict):
-        color information used by the plot
-    font_sz (dict):
-        dictionary of plot methods
-    perfect (bool):
-        determines whether a "perfect" classifier curve is displayed
-    perfect_kwargs (dict):
-        parameters for the perfect classifier for precision/recall curves
-    prob_type (str):
-        model type, i.e. classification or regression
-
-    Methods
-    -------
-    get_legend_labels(legend_labels)
-        Renders the legend labels on the plot
-    plot(evaluation, plots, num_classes, perfect, baseline, legend_labels)
-        Generates the evalation plot
-    """
-
-    # dict of plot methods
-    font_sz = {
-        "xl": 16,  # Plot type title
-        "l": 14,  # Individual plot title (name of model)
-        "m": 12,  # Axis titles
-        "s": 10,  # Axis labels
-        "xs": 8,
-    }  # test within the plot
-
-    baseline_kwargs = {"ls": "--", "c": ".2"}  # lw = ??
-    perfect_kwargs = {"label": "Perfect Classifier", "ls": "--", "color": "gold"}
-    perfect = None
-    baseline = None
-    prob_type = None
-    color_wheel = ["teal", "blueviolet", "forestgreen", "peru", "y", "dodgerblue", "r"]
-
-    _pretty_titles_map = {
-        "normalized_confusion_matrix": "Normalized Confusion Matrix",
-        "lift_chart": "Lift Chart",
-        "gain_chart": "Gain Chart",
-        "ks_statistics": "KS Statistics",
-        "residuals_qq": "Residuals Q-Q Plot",
-        "residuals_vs_predicted": "Residuals vs Predicted",
-        "residuals_vs_observed": "Residuals vs Observed",
-        "observed_vs_predicted": "Observed vs Predicted",
-        "precision_by_label": "Precision by Label",
-        "recall_by_label": "Recall by Label",
-        "f1_by_label": "F1 by Label",
-        "jaccard_by_label": "Jaccard by Label",
-        "pr_curve": "PR Curve",
-        "roc_curve": "ROC Curve",
-        "pr_and_roc_curve": "PR Curve, ROC Curve",
-        "lift_and_gain_chart": "Lift Chart, Gain Chart",
-    }
-
-    _ugly_titles_map = {v: k for k, v in _pretty_titles_map.items()}
-
-    double_overlay_plots = ["pr_and_roc_curve", "lift_and_gain_chart"]
-    single_overlay_plots = ["lift_chart", "gain_chart", "roc_curve", "pr_curve"]
-
-    _bin_plots = [
-        "pr_curve",
-        "roc_curve",
-        "lift_chart",
-        "gain_chart",
-        "normalized_confusion_matrix",
-    ]
-    _multi_plots = [
-        "normalized_confusion_matrix",
-        "roc_curve",
-        "pr_curve",
-        "precision_by_label",
-        "recall_by_label",
-        "f1_by_label",
-        "jaccard_by_label",
-    ]
-    _reg_plots = [
-        "observed_vs_predicted",
-        "residuals_qq",
-        "residuals_vs_predicted",
-        "residuals_vs_observed",
-    ]
-
-    # list of detailed descriptions of each plot type for every classification type, can be extended when adding more metrics
-    _bin_plots_details = """
-    In pattern recognition, information retrieval and binary classification, precision (also called positive predictive value)
-    is the fraction of relevant instances among the retrieved instances, while recall (also known as sensitivity) is the
-    fraction of relevant instances that have been retrieved over the total amount of relevant instances. \n
-    A receiver operating characteristic curve, or ROC curve, is a graphical plot that illustrates the diagnostic ability of a
-    binary classifier system as its discrimination threshold is varied. \n
-    In data mining and association rule learning, lift is a measure of the performance of a targeting model (association rule)
-    at predicting or classifying cases as having an enhanced response (with respect to the population as a whole), measured
-    against a random choice targeting model. \n
-    A gain graph is a graph whose edges are labelled 'invertibly', or 'orientably', by elements of a group G. \n
-    In the field of machine learning and specifically the problem of statistical classification, a confusion matrix, also known
-    as an error matrix, is a specific table layout that allows visualization of the performance of an algorithm, typically a
-    supervised learning one (in unsupervised learning it is usually called a matching matrix).
-    """
-    # Removed for now:
-    # Kuiper 's test (ks_statistics) is used in statistics to test that whether a given distribution, or family of
-    # distributions, is contradicted by evidence from a sample of data. \n
-
-    _multi_plots_details = """
-    In the field of machine learning and specifically the problem of statistical classification, a confusion matrix, also known
-    as an error matrix, is a specific table layout that allows visualization of the performance of an algorithm, typically a
-    supervised learning one (in unsupervised learning it is usually called a matching matrix). \n
-    A receiver operating characteristic curve, or ROC curve, is a graphical plot that illustrates the diagnostic ability of a
-    binary classifier system as its discrimination threshold is varied. \n
-    In pattern recognition, information retrieval and binary classification, precision (also called positive predictive value)
-    is the fraction of relevant instances among the retrieved instances, while recall (also known as sensitivity) is the
-    fraction of relevant instances that have been retrieved over the total amount of relevant instances. \n
-    In statistical analysis of binary classification, the F1 score (also F-score or F-measure) is a measure of a test's accuracy.
-    It considers both the precision p and the recall r of the test to compute the score: p is the number of correct positive results
-    divided by the number of all positive results returned by the classifier, and r is the number of correct positive results divided
-    by the number of all relevant samples (all samples that should have been identified as positive). The F1 score is the harmonic mean
-    of the precision and recall, where an F1 score reaches its best value at 1 (perfect precision and recall) and worst at 0. \n
-    The Jaccard index, also known as Intersection over Union and the Jaccard similarity coefficient, is a statistic used for gauging
-    the similarity and diversity of sample sets. The Jaccard coefficient measures similarity between finite sample sets, and is defined
-    as the size of the intersection divided by the size of the union of the sample sets
-    """
-    _reg_plots_details = """
-    In statistics, a Q–Q (quantile-quantile) plot is a probability plot, which is a graphical method for comparing two probability distributions
-    by plotting their quantiles against each other. \n
-    In statistics and optimization, errors and residuals are two closely related and easily confused measures of the deviation of an observed value
-    of an element of a statistical sample from its "theoretical value". The error (or disturbance) of an observed value is the deviation of the
-    observed value from the (unobservable) true value of a quantity of interest (for example, a population mean), and the residual of an observed
-    value is the difference between the observed value and the estimated value of the quantity of interest (for example, a sample mean).
-    """
-
-    @classmethod
-    def _get_formatted_title(cls, title, max_len=MAX_TITLE_LEN):
-        return title if len(title) < max_len + 3 else title[:max_len] + "..."
-
-    @classmethod
-    def get_legend_labels(cls, legend_labels):
-        """Gets the legend labels, resolves any conflicts such as length, and renders
-        the labels for the plot
-
-        Parameters
-        ----------
-        legend_labels (dict):
-            key/value dictionary containing legend label data
-
-        Returns
-        -------
-        Nothing
-
-        Examples
-        --------
-
-        EvaluationPlot.get_legend_labels({'class_0': 'green', 'class_1': 'yellow', 'class_2': 'red'})
-        """
-
-        @runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
-        @runtime_dependency(
-            module="ipywidgets", object="HTML", install_from=OptionalDependency.NOTEBOOK
-        )
-        def render_legend_labels(label_dict):
-            encodings = pd.DataFrame(
-                pd.Series(label_dict, index=label_dict.keys()),
-                columns=["Shortened labels"],
-            )
-            from IPython.core.display import display, HTML
-
-            display(
-                HTML(
-                    encodings.style.format(precision=4)
-                    .set_properties(**{"text-align": "center"})
-                    .set_table_styles(
-                        [dict(selector="", props=[("text-align", "center")])]
-                    )
-                    .set_table_attributes("class=table")
-                    .set_caption(
-                        '<div align="left"><b style="font-size:20px;">'
-                        + "Legend for labels of the target feature:</b></div>"
-                    )
-                    .to_html()
-                )
-            )
-
-        if legend_labels is not None:
-            # CAUTION: cls.classes is a list of strings. Make sure users know that labels are all converted to strings.
-            if isinstance(legend_labels, dict) and set(cls.classes).issubset(
-                set(legend_labels.keys())
-            ):
-                render_legend_labels(legend_labels)
-                cls.legend_labels = legend_labels
-            else:
-                logger.error(
-                    "The provided `legend_labels` is either not a Python dict or does not possess all possible class labels."
-                )
-            return
-
-        # try to remove leading words
-        def _check_for_redundant_words(label_vec, prefix, max_len=MAX_LEGEND_LEN):
-            words_found = False
-            classes = [lab for lab in label_vec]
-            while len(set([lab.split()[0] for lab in classes])) <= 1:
-                # remove that word from vec, and add it to prefix
-                additional_prefix = classes[0].split()[0]
-                prefix = (
-                    prefix[:-3] + additional_prefix + " ..."
-                    if words_found
-                    else additional_prefix + " ..."
-                )
-                classes = [lab[len(additional_prefix) + 1 :] for lab in classes]
-                words_found = True
-
-            classes = (
-                classes if words_found else [label[max_len:] for label in label_vec]
-            )
-            return classes, prefix
-
-        # returns mapping from real labels to psuedo-labels, when psuedo-labels are not the first X letter
-        def _resolve_conflict(label_vec, prefix, max_len=MAX_LEGEND_LEN):
-            classes, prefix = _check_for_redundant_words(label_vec, prefix)
-            label_dict = _get_labels(classes, max_len=max_len)
-            resolved = {}
-            for orig, new in label_dict.items():
-                resolved[prefix[:-3] + orig] = "..." + new if new[:3] != "..." else new
-            return resolved
-
-        # returns mapping from provided list of strings, to short, unique substrings
-        def _get_labels(classes, max_len=MAX_LEGEND_LEN):
-            conflict_dict = {}
-            for label in classes:
-                prefix = label if len(label) < max_len + 3 else label[:max_len] + "..."
-                if conflict_dict.get(prefix, None) is None:
-                    conflict_dict[prefix] = [label]
-                else:
-                    conflict_dict[prefix].append(label)
-            out = {}
-            for k, v in conflict_dict.items():
-                if len(v) == 1:
-                    out[v[0]] = k
-                else:
-                    resolved = _resolve_conflict(v, k, max_len=MAX_LEGEND_LEN)
-                    out.update(resolved)
-            return out
-
-        cls.legend_labels = _get_labels(cls.classes)
-        if set(cls.legend_labels.keys()) != set(cls.legend_labels.values()):
-            logger.info(
-                f"Class labels greater than {MAX_CHARACTERS_LEN} characters have been truncated. "
-                "Use the `legend_labels` parameter to define labels."
-            )
-            render_legend_labels(cls.legend_labels)
-
-    # evaluation is a DataFrame with models as columns and metrics as rows
-    @classmethod
-    def plot(
-        cls,
-        evaluation,
-        plots,
-        num_classes,
-        perfect=False,
-        baseline=True,
-        legend_labels=None,
-    ):
-        """Generates the evaluation plot
-
-        Parameters
-        ----------
-        evaluation (DataFrame):
-            DataFrame with models as columns and metrics as rows.
-        plots (str):
-            The plot type based on class attribute `prob_type`.
-        num_classes (int):
-            The number of classes for the model.
-        perfect (bool, optional):
-            Whether to display the curve of a perfect classifier. Default value is `False`.
-        baseline (bool, optional):
-            Whether to display the curve of the baseline, featureless model. Default value is `True`.
-        legend_labels (dict, optional):
-            Legend labels dictionary. Default value is `None`. If legend_labels not specified class names will be used for plots.
-
-        Returns
-        -------
-        Nothing
-        """
-
-        cls.perfect = perfect
-        cls.baseline = baseline
-        # get plots to show
-        if num_classes == 2:
-            cls.prob_type = "_bin"
-        elif num_classes > 2:
-            cls.prob_type = "_multi"
-        else:
-            cls.prob_type = "_reg"
-        plot_details = getattr(cls, cls.prob_type + "_plots_details")
-        if plots is None:
-            plots = getattr(cls, cls.prob_type + "_plots")
-            logger.info(
-                "Showing plot types: {}.".format(
-                    ", ".join(
-                        [
-                            "{}".format(EvaluationPlot._pretty_titles_map[str(p)])
-                            for p in plots
-                        ]
-                    ),
-                    ", ".join(["{}".format(x) for x in map(str, plots)]),
-                )
-            )
-            logger.info(plot_details)
-
-        if cls.prob_type == "_bin":
-            if "lift_chart" in plots and "gain_chart" in plots:
-                plots.remove("lift_chart")
-                plots.remove("gain_chart")
-                plots.insert(0, "lift_and_gain_chart")
-
-            if "roc_curve" in plots and "pr_curve" in plots:
-                plots.remove("roc_curve")
-                plots.remove("pr_curve")
-                plots.insert(0, "pr_and_roc_curve")
-        elif cls.prob_type == "_multi":
-            if (
-                "normalized_confusion_matrix" in plots
-                and len(evaluation[evaluation.columns[0]]["classes"])
-                >= MAX_PLOTTING_CLASSES
-            ):
-                logger.error(
-                    f"Evaluation plotting is not yet supported for multiclass problems with {MAX_PLOTTING_CLASSES} or more classes."
-                )
-                plots = []
-        classes = evaluation[evaluation.columns[0]]["classes"]
-        if classes is not None:
-            # CAUTION: class labels are converted to strings here.
-            # If users are passing in legend_labels, they have to use strings as well.
-            # Otherwise get_legend_labels() will complaint.
-            # If users are not passing in legend_labels, get_legend_labels() generates them from cls.classes.
-            # cls.legend_labels are assigned/created in cls.get_lengend_labels() and contain only strings as keys.
-            cls.classes = [str(c) for c in classes]
-            cls.get_legend_labels(legend_labels)
-
-        mpl.style.use("default")
-        html_raw = []
-        for i, plot_type in enumerate(plots):
-            fig_title, fig = None, None
-            try:
-                fig_title, ax_title = plt.subplots(1, 1, figsize=(18, 0.5), dpi=144)
-                ax_title.text(
-                    0.5,
-                    0.5,
-                    cls._pretty_titles_map[plot_type],
-                    fontsize=16,
-                    fontweight="semibold",
-                    horizontalalignment="center",
-                    verticalalignment="center",
-                    transform=ax_title.transAxes,
-                )
-                ax_title.axis("off")
-                html_raw.append(_fig_to_html(fig_title))
-                if cls.prob_type == "_bin" and plot_type in cls.double_overlay_plots:
-                    fig, ax = plt.subplots(1, 2, figsize=(12, 4.5), dpi=144)
-                elif cls.prob_type == "_bin" and plot_type in ["roc_curve", "pr_curve"]:
-                    fig, axs = plt.subplots(1, 2, figsize=(12, 4.5), dpi=144)
-                    axs[1].axis("off")
-                    ax = [axs[0]]
-                elif cls.prob_type == "_bin" and plot_type in [
-                    "lift_chart",
-                    "gain_chart",
-                ]:
-                    fig, axs = plt.subplots(1, 2, figsize=(12, 4.5), dpi=144)
-                    axs[1].axis("off")
-                    ax = axs[0]
-                else:
-                    nrows = math.ceil(len(evaluation.columns) / MAX_PLOTS_PER_ROW)
-                    fig, ax = plt.subplots(
-                        nrows, MAX_PLOTS_PER_ROW, figsize=(10, 4 * nrows), dpi=144
-                    )  # 10, 3.5
-                    ax = ax.flatten()
-                getattr(cls, "_" + plot_type)(ax, evaluation)
-                fig.tight_layout()
-                html_raw.append(_fig_to_html(fig))
-            except KeyError as e:
-                try:
-                    if fig_title:
-                        plt.close(fig=fig_title)
-                    if fig:
-                        plt.close(fig=fig)
-                except:
-                    pass
-                logger.warning(
-                    f"Evaluator was not able to plot "
-                    f"{cls._pretty_titles_map.get(plot_type, plot_type)}, because the relevant "
-                    f"metrics had complications. Ensure that `predict` and `predict_proba` "
-                    f"are valid."
-                )
-        return html_raw
-
-    @classmethod
-    def _lift_and_gain_chart(cls, ax, evaluation):
-        cls._lift_chart(ax[0], evaluation)
-        cls._gain_chart(ax[1], evaluation)
-
-    @classmethod
-    def _lift_chart(cls, ax, evaluation):
-        for mod_name, col in evaluation.items():
-            if col["y_score"] is not None:
-                ax.plot(
-                    col["percentages"][1:],
-                    [1] + list(col["lift"]),
-                    label=cls._get_formatted_title(mod_name),
-                )
-        if cls.baseline:
-            ax.plot([-10, 110], [1, 1], **cls.baseline_kwargs)
-        if cls.perfect:
-            perf_idx = next(
-                idx
-                for idx, scores in enumerate(evaluation.loc["y_score"])
-                if scores is not None
-            )
-            ax.plot(
-                evaluation.loc["percentages"][perf_idx][1:],
-                [1] + list(evaluation.loc["perfect_lift"][perf_idx]),
-                **cls.perfect_kwargs,
-            )
-        ax.legend(loc="upper right", frameon=False)
-        ax.set_xlabel("Percentage of Population", fontsize=12)
-        ax.set_ylabel("Lift", fontsize=12)
-        ax.set_title("Lift Chart", y=1.08, fontsize=14)
-        ax.grid(linewidth=0.2, which="both")
-        ax.set_xlim([-10, 110])
-
-    @classmethod
-    def _gain_chart(cls, ax, evaluation):
-        for mod_name, col in evaluation.items():
-            if col["y_score"] is not None:
-                ax.plot(
-                    col["percentages"],
-                    list(col["cumulative_gain"]),
-                    label=cls._get_formatted_title(mod_name),
-                )
-        if cls.baseline:
-            ax.plot([-10, 110], [-10, 110], **cls.baseline_kwargs)
-        if cls.perfect:
-            perf_idx = next(
-                idx
-                for idx, scores in enumerate(evaluation.loc["y_score"])
-                if scores is not None
-            )
-            ax.plot(
-                evaluation.loc["percentages"][perf_idx],
-                evaluation.loc["perfect_gain"][perf_idx],
-                **cls.perfect_kwargs,
-            )
-        ax.legend(loc="lower right", frameon=False)
-        ax.set_xlabel("Percentage of Population", fontsize=12)
-        ax.set_ylabel("Percentage of Positive Class", fontsize=12)
-        ax.set_title("Gain Chart", y=1.08, fontsize=14)
-        ax.grid(linewidth=0.2, which="both")
-        ax.set_xlim([-10, 110])
-        ax.set_ylim([-10, 110])
-
-    @classmethod
-    def _pr_and_roc_curve(cls, ax, evaluation):
-        cls._pr_curve([ax[0]], evaluation)
-        cls._roc_curve([ax[1]], evaluation)
-
-    @classmethod
-    def _pr_curve(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-            if cls.prob_type == "_bin":
-                for mod_name, col in evaluation.items():
-                    if col["y_score"] is not None:
-                        ax.plot(
-                            col["recall_values"],
-                            col["precision_values"],
-                            label="%s (Precision: %s)"
-                            % (
-                                cls._get_formatted_title(mod_name),
-                                "{:.3f}".format(col["precision"]),
-                            ),
-                        )
-                        ax.plot(
-                            *col["pr_best_model_score"],
-                            color=ax.get_lines()[-1].get_color(),
-                            marker="*",
-                        )
-            else:
-                model_name = evaluation.columns[i]
-                mod = evaluation[model_name]
-                if mod["y_score"] is not None:
-                    for j, lab in enumerate(mod.classes):
-                        # cls.legend_labels contains only strings as keys.
-                        lab = str(lab)
-                        ax.plot(
-                            mod["recall_values"][j],
-                            mod["precision_values"][j],
-                            label="%s (Precision: %s)"
-                            % (
-                                cls.legend_labels[lab],
-                                "{:.3f}".format(mod["precision_by_label"][j]),
-                            ),
-                        )
-                        ax.plot(
-                            *mod["pr_best_model_score"][j],
-                            color=ax.get_lines()[-1].get_color(),
-                            marker="*",
-                        )
-
-            ax.set_xlabel("Recall", fontsize=12)
-            ax.set_ylabel("Precision", fontsize=12)
-            ax.set_title("Precision Recall Curve", y=1.08, fontsize=14)
-            ax.grid(linewidth=0.2, which="both")
-            ax.set_xlim([-0.1, 1.1])
-            ax.set_ylim([-0.1, 1.1])
-            handles, labels = ax.get_legend_handles_labels()
-            star = mlines.Line2D(
-                [],
-                [],
-                color="black",
-                marker="*",
-                linestyle="None",
-                markersize=5,
-                label="Minimum Error Rate",
-            )
-            handles.append(star)
-            labels.append("Minimum Error Rate")
-            ax.legend(
-                loc="upper right",
-                labels=labels,
-                handles=handles,
-                frameon=False,
-                fontsize="x-small",
-            )
-
-    @classmethod
-    def _roc_curve(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-            if cls.prob_type == "_bin":
-                for mod_name, col in evaluation.items():
-                    if col["y_score"] is not None:
-                        ax.plot(
-                            col["false_positive_rate"],
-                            col["true_positive_rate"],
-                            label="%s (AUC: %s)"
-                            % (
-                                cls._get_formatted_title(mod_name),
-                                "{:.3f}".format(col["auc"]),
-                            ),
-                        )
-                        ax.plot(
-                            *col["roc_best_model_score"],
-                            color=ax.get_lines()[-1].get_color(),
-                            marker="*",
-                        )
-            else:
-                model_name = evaluation.columns[i]
-                mod = evaluation[model_name]
-                if mod["y_score"] is not None:
-                    for j, lab in enumerate(mod.classes):
-                        # cls.legend_labels contains only strings as keys.
-                        lab = str(lab)
-                        ax.plot(
-                            mod["fpr_by_label"][j],
-                            mod["tpr_by_label"][j],
-                            label="%s (AUC: %s)"
-                            % (cls.legend_labels[lab], "{:.3f}".format(mod["auc"][j])),
-                        )
-                        ax.plot(
-                            *mod["roc_best_model_score"][j],
-                            color=ax.get_lines()[-1].get_color(),
-                            marker="*",
-                        )
-            if cls.baseline:
-                ax.plot([-0.1, 1.1], [-0.1, 1.1], **cls.baseline_kwargs)
-            ax.set_xlabel("False Positive Rate", fontsize=12)
-            ax.set_ylabel("True Positive Rate", fontsize=12)
-            ax.set_title("ROC Curve", y=1.08, fontsize=14)
-            ax.grid(linewidth=0.2, which="both")
-            ax.set_xlim([-0.1, 1.1])
-            ax.set_ylim([-0.1, 1.1])
-            handles, labels = ax.get_legend_handles_labels()
-            star = mlines.Line2D(
-                [],
-                [],
-                color="black",
-                marker="*",
-                linestyle="None",
-                markersize=5,
-                label="Youden's J Statistic",
-            )
-            handles.append(star)
-            labels.append("Youden's J Statistic")
-            ax.legend(
-                loc="lower right",
-                labels=labels,
-                handles=handles,
-                frameon=False,
-                fontsize="x-small",
-            )
-
-    @classmethod
-    def _ks_statistics(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-            model_name = evaluation.columns[i]
-            mod = evaluation[model_name]
-
-            ax.set_title(model_name, fontsize=14)
-            if mod["y_score"] is not None:
-                ax.plot(
-                    mod["ks_thresholds"],
-                    mod["ks_pct1"],
-                    lw=3,
-                    label=mod["ks_labels"][0],
-                )
-                ax.plot(
-                    mod["ks_thresholds"],
-                    mod["ks_pct2"],
-                    lw=3,
-                    label=mod["ks_labels"][1],
-                )
-                if cls.baseline:
-                    idx = np.where(mod["ks_thresholds"] == mod["max_distance_at"])[0][0]
-                    ax.axvline(
-                        mod["max_distance_at"],
-                        *sorted([mod["ks_pct1"][idx], mod["ks_pct2"][idx]]),
-                        label="KS Statistic: {:.3f} at {:.3f}".format(
-                            mod["ks_statistic"], mod["max_distance_at"]
-                        ),
-                        linestyle="--",
-                        color=".2",
-                    )
-
-            ax.set_xlim([0.0, 1.0])
-            ax.set_ylim([0.0, 1.0])
-
-            ax.set_xlabel("Threshold", fontsize=12)
-            ax.set_ylabel("Percentage below threshold", fontsize=12)
-            ax.tick_params(labelsize=10)
-            ax.legend(loc="lower right", fontsize=8)
-
-    @classmethod
-    def _pretty_barh(
-        cls, ax, x, y, axis_labels=None, title=None, axis_lim=None, plot_kwargs=None
-    ):
-        # cls.legend_labels contains only strings as keys.
-        new_lab = [cls.legend_labels[str(item)] for item in x]
-        ax.barh(
-            new_lab,
-            y,
-            color=["teal", "blueviolet", "forestgreen", "peru", "y", "dodgerblue", "r"],
-        )
-        for j, v in enumerate(y):
-            ax.annotate("{:.3f}".format(v), xy=(v / 2, j), va="center", ha="left")
-        if axis_labels:
-            if axis_labels[0]:
-                ax.set_xlabel(axis_labels[0], fontsize=12)
-            if axis_labels[1]:
-                ax.set_ylabel(axis_labels[1], fontsize=12)
-        if title:
-            title = cls._get_formatted_title(title)
-            ax.set_title(title, y=1.08, fontsize=14)
-        if axis_lim:
-            ax.set_xlim(axis_lim)
-
-    @classmethod
-    def _precision_by_label(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i < n_models:
-                col = evaluation.columns[i]
-                cls._pretty_barh(
-                    ax,
-                    evaluation[col]["classes"],
-                    evaluation[col]["precision_by_label"],
-                    axis_lim=[0, 1],
-                    axis_labels=["Precision", None],
-                    title=col,
-                )
-            else:
-                ax.axis("off")
-
-    @classmethod
-    def _recall_by_label(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i < n_models:
-                col = evaluation.columns[i]
-                cls._pretty_barh(
-                    ax,
-                    evaluation[col]["classes"],
-                    evaluation[col]["recall_by_label"],
-                    axis_lim=[0, 1],
-                    axis_labels=["Recall", None],
-                    title=col,
-                )
-            else:
-                ax.axis("off")
-
-    @classmethod
-    def _f1_by_label(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i < n_models:
-                col = evaluation.columns[i]
-                cls._pretty_barh(
-                    ax,
-                    evaluation[col]["classes"],
-                    evaluation[col]["f1_by_label"],
-                    axis_lim=[0, 1],
-                    axis_labels=["F1 Score", None],
-                    title=col,
-                )
-            else:
-                ax.axis("off")
-
-    @classmethod
-    def _jaccard_by_label(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i < n_models:
-                col = evaluation.columns[i]
-                cls._pretty_barh(
-                    ax,
-                    evaluation[col]["classes"],
-                    evaluation[col]["jaccard_by_label"],
-                    axis_lim=[0, 1],
-                    axis_labels=["Jaccard Score", None],
-                    title=col,
-                )
-            else:
-                ax.axis("off")
-
-    @classmethod
-    def _pretty_scatter(
-        cls,
-        ax,
-        x,
-        y,
-        s=5,
-        alpha=1.0,
-        title=None,
-        legend=False,
-        axis_labels=None,
-        axis_lim=None,
-        grid=True,
-        label=None,
-        plot_kwargs=None,
-    ):
-        if plot_kwargs is None:
-            plot_kwargs = {}
-        ax.scatter(x, y, s=s, label=label, marker="o", alpha=alpha, **plot_kwargs)
-        ax.xaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-        ax.yaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-        if legend:
-            ax.legend(frameon=False)
-        if axis_labels:
-            ax.set_xlabel(axis_labels[0])
-            ax.set_ylabel(axis_labels[1])
-        if title:
-            ax.set_title(title, y=1.08, fontsize=14)
-        if grid:
-            ax.grid(linewidth=0.2)
-        if axis_lim:
-            if axis_lim[0]:
-                ax.set_xlim(axis_lim[0])
-            if axis_lim[1]:
-                ax.set_ylim(axis_lim[1])
-
-    @classmethod
-    def _top_2_features(cls, axs, evaluation):
-        pass
-
-    @classmethod
-    def _residuals_qq(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-            model_name = evaluation.columns[i]
-            mod = evaluation[model_name]
-            # getattr(ax, self.plot_method)(self.x, y, **self.plot_kwargs, color='#4a91c2', label=label)
-            cls._pretty_scatter(
-                ax,
-                mod["norm_quantiles"],
-                mod["residual_quantiles"],
-                title=model_name,
-                axis_lim=[(-2.7, 2.7), (-3.1, 3.1)],
-                axis_labels=["Theoretical Quantiles", "Sample Quantiles"],
-            )
-            if cls.baseline:
-                ax.plot((-100, 100), (-100, 100), **cls.baseline_kwargs)
-
-    @classmethod
-    def _residuals_vs_predicted(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-            model_name = evaluation.columns[i]
-            mod = evaluation[model_name]
-            y_pred = np.asarray(mod["y_pred"])
-            resid = np.asarray(mod["residuals"])
-            cls._pretty_scatter(
-                ax,
-                y_pred,
-                resid,
-                s=4,
-                alpha=0.5,
-                title=model_name,
-                axis_labels=["Predicted Values", "Residuals"],
-            )
-            x_lim = (
-                y_pred.min() - y_pred.min() * 0.05,
-                y_pred.max() + y_pred.max() * 0.05,
-            )
-            if cls.baseline:
-                ax.plot(x_lim, (0, 0), **cls.baseline_kwargs)
-            ax.set_xlim(x_lim)
-            ax.xaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-            ax.yaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-
-    @classmethod
-    def _residuals_vs_observed(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-            model_name = evaluation.columns[i]
-            mod = evaluation[model_name]
-            y_true = np.asarray(mod["y_true"])
-            cls._pretty_scatter(
-                ax,
-                y_true,
-                mod["residuals"],
-                s=4,
-                alpha=0.5,
-                title=model_name,
-                axis_labels=["Observed Values", "Residuals"],
-            )
-            x_lim = (
-                y_true.min() - y_true.min() * 0.05,
-                y_true.max() + y_true.max() * 0.05,
-            )
-            if cls.baseline:
-                ax.plot(x_lim, (0, 0), **cls.baseline_kwargs)
-            ax.set_xlim(x_lim)
-            ax.xaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-            ax.yaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-
-    @classmethod
-    def _observed_vs_predicted(cls, axs, evaluation):
-        n_models = len(evaluation.columns)
-        for i, ax in enumerate(axs):
-            if i >= n_models:
-                ax.axis("off")
-                return
-
-            model_name = evaluation.columns[i]
-            mod = evaluation[model_name]
-            ax.scatter(mod["y_true"], mod["y_pred"], s=4, marker="o", alpha=0.5)
-
-            y_true = np.asarray(mod["y_true"])
-
-            yt_min = y_true.min()
-            yt_max = y_true.max()
-
-            x_lim = (yt_min - yt_min * 0.05, yt_max + yt_max * 0.05)
-            if cls.baseline:
-                ax.plot(x_lim, x_lim, **cls.baseline_kwargs)
-            ax.set_xlabel("Observed Values")
-            ax.set_ylabel("Predicted Values")
-            ax.set_title(model_name, y=1.08, fontsize=14)
-            ax.grid(linewidth=0.2)
-            ax.set_xlim(x_lim)
-
-            ax.xaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-            ax.yaxis.set_major_formatter(FormatStrFormatter("%.2f"))
-
-    @classmethod
-    def _normalized_confusion_matrix(cls, axs, evaluation):
-        for model_num, ax in enumerate(axs):
-            if model_num >= len(evaluation.columns):
-                ax.axis("off")
-                return
-            model_name = evaluation.columns[model_num]
-            mod = evaluation[model_name]
-            if cls.prob_type == "_bin":
-                labels = [str(lab == mod["positive_class"]) for lab in mod["classes"]]
-            else:
-                labels = cls.legend_labels.values()
-
-            raw_cm = mod["raw_confusion_matrix"]
-            cm = np.asarray(mod["confusion_matrix"])
-
-            ax.set_title(
-                "%s\n" % cls._get_formatted_title(model_name), y=1.08, fontsize=14
-            )
-            ax.imshow(cm, interpolation="nearest", cmap="BuGn")
-            x_tick_marks = np.arange(len(labels))
-            y_tick_marks = np.arange(len(labels))
-            ax.set_xticks(x_tick_marks)
-            ax.set_yticks(y_tick_marks)
-
-            ax.set_xticklabels(labels, rotation=90, fontsize=10)
-            ax.set_yticklabels(labels, fontsize=10)
-
-            for i, j in itertools.product(
-                range(raw_cm.shape[0]), range(raw_cm.shape[1])
-            ):
-                ax.text(
-                    j,
-                    i,
-                    "%s [%s]" % (round(cm[i][j], 3), raw_cm[i, j]),
-                    horizontalalignment="center",
-                    verticalalignment="center",
-                    rotation=45,
-                    fontsize=max(3, 10 - max(raw_cm.shape[0], raw_cm.shape[1])),
-                    color="white" if cm[i, j] > 0.5 else "black",
-                )
-
-            ax.set_ylabel("True label", fontsize=10)
-            ax.set_xlabel("Predicted label", fontsize=10)
-            ax.grid(False)