oodeel 0.4.0__py3-none-any.whl

oodeel/eval/metrics.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import re
+
+import numpy as np
+import sklearn
+
+from ..types import Optional
+from ..types import Tuple
+from ..types import Union
+
+
+def bench_metrics(
+    scores: Union[np.ndarray, tuple],
+    labels: Optional[np.ndarray] = None,
+    in_value: Optional[int] = 0,
+    out_value: Optional[int] = 1,
+    metrics: Optional[list] = ["auroc", "fpr95tpr"],
+    threshold: Optional[float] = None,
+    step: Optional[int] = 4,
+) -> dict:
+    """Compute various common metrics from the OOD detector scores:
+    AUROC, FPR95TPR (or any other similar metric relative to confusion matrix),
+    Detection accuracy and sklearn.metric metrics
+
+    Args:
+        scores (Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]): scores output of
+            the OOD detector to evaluate. If a tuple is provided,
+            the first array is considered in-distribution scores, and the second
+            is considered out-of-distribution scores.
+        labels (Optional[np.ndarray], optional): labels denoting oodness. When scores is
+            a tuple, this argument and the following in_value and out_value are not
+            used. If scores is a np.ndarray, labels are required with in_value and
+            out_value if different from their default values.
+            Defaults to None.
+        in_value (Optional[int], optional): ood label value for in-distribution data.
+            Automatically assigned 0 if it is not the case.
+            Defaults to 0.
+        out_value (Optional[int], optional): ood label value for out-of-distribution
+            data. Automatically assigned 1 if it is not the case.
+            Defaults to 1.
+        metrics (Optional[List[str]], optional): list of metrics to compute. Can pass
+            any metric name from sklearn.metric or among "detect_acc" and
+            "<aaa><XX><bbb>" where <aaa> and <bbb> are in ["fpr", "tpr", "fnr", "tnr"]
+            and <XX> is an integer between 1 and 99. Defaults to ["auroc", "fpr95tpr"].
+        threshold (Optional[float], optional): Threshold to use when using
+            threshold-dependent metrics. Defaults to None.
+        step (Optional[int], optional): integration step (wrt percentile).
+            Only used for auroc and fpr95tpr. Defaults to 4.
+
+    Returns:
+        dict: Dictionnary of metrics
+    """
+    metrics_dict = {}
+
+    if isinstance(scores, np.ndarray):
+        assert labels is not None, (
+            "Provide labels with scores, or provide a tuple of in-distribution "
+            "and out-of-distribution scores arrays"
+        )
+        labels = np.copy(labels)  # to avoid mutable np.array to be modified
+        labels[labels == in_value] = 0
+        labels[labels == out_value] = 1
+    elif isinstance(scores, tuple):
+        scores_in, scores_out = scores
+        scores = np.concatenate([scores_in, scores_out])
+        labels = np.concatenate([scores_in * 0, scores_out * 0 + 1])
+
+    fpr, tpr, fnr, tnr, acc = get_curve(scores, labels, step)
+
+    for metric in metrics:
+        if isinstance(metric, str):
+            if metric == "auroc":
+                if np.__version__ >= "2.0.0":
+                    auroc = -np.trapezoid(1.0 - fpr, tpr)
+                else:
+                    auroc = -np.trapz(1.0 - fpr, tpr)
+                metrics_dict["auroc"] = auroc
+
+            elif metric == "detect_acc":
+                metrics_dict["detect_acc"] = np.max(acc)
+
+            # compute <aaa><XX><bbb> metrics (check docstring for more info)
+            elif (
+                re.search(r"^(fpr|tpr|fnr|tnr)(\d{1,2})(fpr|tpr|fnr|tnr)$", metric)
+                is not None
+            ):
+                count_1_str, thr, count_2_str = re.match(
+                    pattern=r"^(fpr|tpr|fnr|tnr)(\d{1,2})(fpr|tpr|fnr|tnr)$",
+                    string=metric,
+                ).groups()
+                thr = int(thr)
+                count_1, count_2 = locals()[count_1_str], locals()[count_2_str]
+                for i, c2 in enumerate(count_2):
+                    if (count_2_str in ["fpr", "tpr"] and c2 < thr / 100) or (
+                        count_2_str in ["tnr", "fnr"] and c2 > thr / 100
+                    ):
+                        ind = i
+                        break
+                metrics_dict[metric] = count_1[ind]
+
+        elif metric.__name__ in sklearn.metrics.__all__:
+            if metric.__name__[:3] == "roc":
+                metrics_dict[metric.__name__] = metric(labels, scores)
+            else:
+                if threshold is None:
+                    print(
+                        f"No threshold is specified for metric {metric.__name__}, "
+                        "skipping"
+                    )
+                else:
+                    oodness = [1 if x > threshold else 0 for x in scores]
+                    metrics_dict[metric.__name__] = metric(labels, oodness)
+
+        else:
+            print(f"Metric {metric.__name__} not implemented, skipping")
+
+    return metrics_dict
+
+
+def get_curve(
+    scores: np.ndarray,
+    labels: np.ndarray,
+    step: Optional[int] = 4,
+    return_raw: Optional[bool] = False,
+) -> Union[Tuple[tuple, tuple], tuple]:
+    """Computes the
+        * true positive rate: TP / (TP + FN),
+        * false positive rate: FP / (FP + TN),
+        * true negative rate: TN / (FP + TN),
+        * false negative rate: FN / (TP + FN),
+        * accuracy: (TN + TP) / (TP + FP + TN + FN),
+    for different threshold values. The values are uniformly
+    distributed among the percentiles, with a step = 4 / scores.shape[0]
+
+    Args:
+        scores (np.ndarray): scores output of the OOD detector to evaluate
+        labels (np.ndarray): 1 if ood else 0
+        step (Optional[int], optional): integration step (wrt percentile).
+            Defaults to 4.
+        return_raw (Optional[bool], optional): To return all the curves
+            or only the rate curves. Defaults to False.
+
+    Returns:
+        Union[Tuple[Tuple[np.ndarray], Tuple[np.ndarray]], Tuple[np.ndarray]]: curves
+    """
+    tpc = np.array([])
+    fpc = np.array([])
+    tnc = np.array([])
+    fnc = np.array([])
+    thresholds = np.sort(scores)
+    for i in range(1, len(scores), step):
+        fp, tp, fn, tn = ftpn(scores, labels, thresholds[i])
+        tpc = np.append(tpc, tp)
+        fpc = np.append(fpc, fp)
+        tnc = np.append(tnc, tn)
+        fnc = np.append(fnc, fn)
+
+    fpr = np.concatenate([[1.0], fpc / (fpc + tnc), [0.0]])
+    tpr = np.concatenate([[1.0], tpc / (tpc + fnc), [0.0]])
+    tnr = np.concatenate([[0.0], tnc / (fpc + tnc), [1.0]])
+    fnr = np.concatenate([[0.0], fnc / (tpc + fnc), [1.0]])
+    acc = (tnc + tpc) / (tpc + fpc + tnc + fnc)
+
+    if return_raw:
+        return (fpc, tpc, fnc, tnc), (fpr, tpr, fnr, tnr, acc)
+    else:
+        return fpr, tpr, fnr, tnr, acc
+
+
+def ftpn(scores: np.ndarray, labels: np.ndarray, threshold: float) -> tuple:
+    """Computes the number of
+        * true positives,
+        * false positives,
+        * true negatives,
+        * false negatives,
+    for a given threshold
+
+    Args:
+        scores (np.ndarray): scores output of the OOD detector to evaluate
+        labels (np.ndarray): 1 if ood else 0
+        threshold (float): threshold to use to consider scores
+            as in-distribution or out-of-distribution
+
+    Returns:
+        Tuple[float]: The four metrics
+    """
+    pos = np.where(scores >= threshold)
+    neg = np.where(scores < threshold)
+    n_pos = len(pos[0])
+    n_neg = len(neg[0])
+
+    tp = np.sum(labels[pos])
+    fp = n_pos - tp
+    fn = np.sum(labels[neg])
+    tn = n_neg - fn
+
+    return fp, tp, fn, tn

oodeel/eval/plots/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from .features import plot_2D_features
+from .features import plot_3D_features
+from .metrics import plot_ood_scores
+from .metrics import plot_roc_curve
+from .plotly import plotly_3D_features

oodeel/eval/plots/features.py

@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+# Copyright IRT Antoine de Saint Exupéry et Université Paul Sabatier Toulouse III - All
+# rights reserved. DEEL is a research program operated by IVADO, IRT Saint Exupéry,
+# CRIAQ and ANITI - https://www.deel.ai/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+import sklearn
+from matplotlib.lines import Line2D
+from packaging.version import parse
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+
+from ...types import Callable
+from ...types import DatasetType
+from ...types import Union
+from ...utils import import_backend_specific_stuff
+
+sns.set_style("darkgrid")
+
+PROJ_DICT = {
+    "TSNE": {
+        "name": "t-SNE",
+        "class": TSNE,
+        "default_kwargs": dict(perplexity=30.0, n_iter=800, random_state=0),
+    },
+    "PCA": {"name": "PCA", "class": PCA, "default_kwargs": dict()},
+}
+
+# check sklearn version: if > 1.5, use max_iter instead of n_iter
+if parse(sklearn.__version__) >= parse("1.5"):
+    n_iter = PROJ_DICT["TSNE"]["default_kwargs"].pop("n_iter")
+    PROJ_DICT["TSNE"]["default_kwargs"]["max_iter"] = n_iter
+
+
+def plot_2D_features(
+    model: Callable,
+    in_dataset: DatasetType,
+    output_layer_id: Union[int, str],
+    out_dataset: DatasetType = None,
+    proj_method: str = "TSNE",
+    max_samples: int = 4000,
+    title: str = None,
+    **proj_kwargs,
+):
+    """Visualize ID and OOD features of a model on a 2D plan using dimensionality
+    reduction methods and matplotlib scatter function. Different projection methods are
+    available: TSNE, PCA.
+
+    Args:
+        model (Callable): Torch or Keras model.
+        in_dataset (DatasetType): In-distribution dataset (torch dataloader or tf
+            dataset) that will be projected on the model feature space.
+        output_layer_id (Union[int, str]): Identifier for the layer to inspect.
+        out_dataset (DatasetType, optional): Out-of-distribution dataset (torch
+            dataloader or tf dataset) that will be projected on the model feature space
+            if not equal to None. Defaults to None.
+        proj_method (str, optional): Projection method for 2d dimensionality reduction.
+            Defaults to "TSNE", alternative: "PCA".
+        max_samples (int, optional): Max samples to display on the scatter plot.
+            Defaults to 4000.
+        title (str, optional): Custom figure title. Defaults to None.
+    """
+
+    _plot_features(
+        model=model,
+        in_dataset=in_dataset,
+        output_layer_id=output_layer_id,
+        out_dataset=out_dataset,
+        proj_method=proj_method,
+        max_samples=max_samples,
+        title=title,
+        n_components=2,
+        **proj_kwargs,
+    )
+
+
+def plot_3D_features(
+    model: Callable,
+    in_dataset: DatasetType,
+    output_layer_id: Union[int, str],
+    out_dataset: DatasetType = None,
+    proj_method: str = "TSNE",
+    max_samples: int = 4000,
+    title: str = None,
+    **proj_kwargs,
+):
+    """Visualize ID and OOD features of a model on a 3D space using dimensionality
+    reduction methods and matplotlib scatter function. Different projection methods are
+    available: TSNE, PCA.
+
+    Args:
+        model (Callable): Torch or Keras model.
+        in_dataset (DatasetType): In-distribution dataset (torch dataloader or tf
+            dataset) that will be projected on the model feature space.
+        output_layer_id (Union[int, str]): Identifier for the layer to inspect.
+        out_dataset (DatasetType, optional): Out-of-distribution dataset (torch
+            dataloader or tf dataset) that will be projected on the model feature space
+            if not equal to None. Defaults to None.
+        proj_method (str, optional): Projection method for 2d dimensionality reduction.
+            Defaults to "TSNE", alternative: "PCA".
+        max_samples (int, optional): Max samples to display on the scatter plot.
+            Defaults to 4000.
+        title (str, optional): Custom figure title. Defaults to None.
+    """
+    _plot_features(
+        model=model,
+        in_dataset=in_dataset,
+        output_layer_id=output_layer_id,
+        out_dataset=out_dataset,
+        proj_method=proj_method,
+        max_samples=max_samples,
+        title=title,
+        n_components=3,
+        **proj_kwargs,
+    )
+
+
+def _plot_features(
+    model: Callable,
+    in_dataset: DatasetType,
+    output_layer_id: Union[int, str],
+    out_dataset: DatasetType = None,
+    proj_method: str = "TSNE",
+    max_samples: int = 4000,
+    title: str = None,
+    n_components: int = 2,
+    **proj_kwargs,
+):
+    """Visualize ID and OOD features of a model on a 2D or 3D space using dimensionality
+    reduction methods and matplotlib scatter function. Different projection methods are
+    available: TSNE, PCA.
+
+    Args:
+        model (Callable): Torch or Keras model.
+        in_dataset (DatasetType): In-distribution dataset (torch dataloader or tf
+            dataset) that will be projected on the model feature space.
+        output_layer_id (Union[int, str]): Identifier for the layer to inspect.
+        out_dataset (DatasetType, optional): Out-of-distribution dataset (torch
+            dataloader or tf dataset) that will be projected on the model feature space
+            if not equal to None. Defaults to None.
+        proj_method (str, optional): Projection method for 2d dimensionality reduction.
+            Defaults to "TSNE", alternative: "PCA".
+        max_samples (int, optional): Max samples to display on the scatter plot.
+            Defaults to 4000.
+        title (str, optional): Custom figure title. If None a default one is provided.
+            Defaults to None.
+    """
+    assert n_components in [2, 3], "The number of components should be 2 or 3"
+    max_samples = max_samples if out_dataset is None else max_samples // 2
+
+    # feature extractor
+    _, _, op, FeatureExtractorClass = import_backend_specific_stuff(model)
+    feature_extractor = FeatureExtractorClass(model, [output_layer_id])
+
+    # === extract id features ===
+    # features
+    in_features, _ = feature_extractor.predict(in_dataset, numpy_concat=True)
+    in_features = in_features[0].reshape(in_features[0].shape[0], -1)[:max_samples]
+
+    # labels
+    in_labels = []
+    for _, batch_y in in_dataset:
+        in_labels.append(op.convert_to_numpy(batch_y))
+    in_labels = np.concatenate(in_labels)[:max_samples]
+    in_labels_str = list(map(lambda x: f"class {x}", in_labels))
+
+    # === extract ood features ===
+    if out_dataset is not None:
+        # features
+        out_features, _ = feature_extractor.predict(out_dataset, numpy_concat=True)
+        out_features = out_features[0].reshape(out_features[0].shape[0], -1)[
+            :max_samples
+        ]
+
+        # labels
+        out_labels_str = np.array(["unknown"] * len(out_features))
+
+        # concatenate id and ood items
+        features = np.concatenate([out_features, in_features])
+        labels_str = np.concatenate([out_labels_str, in_labels_str])
+        data_type = np.array(
+            ["OOD"] * len(out_labels_str) + ["ID"] * len(in_labels_str)
+        )
+    else:
+        features = in_features
+        labels_str = in_labels_str
+        data_type = np.array(["ID"] * len(in_labels))
+
+    # === project on 2d/3d space using tsne or pca ===
+    proj_class = PROJ_DICT[proj_method]["class"]
+    p_kwargs = PROJ_DICT[proj_method]["default_kwargs"]
+    p_kwargs.update(proj_kwargs)
+    projector = proj_class(
+        n_components=n_components,
+        **p_kwargs,
+    )
+    features_proj = projector.fit_transform(features)
+
+    # === plot 2d/3d features ===
+    features_dim = features.shape[1]
+    method_str = PROJ_DICT[proj_method]["name"]
+    title = (
+        title
+        or f"{method_str} {n_components}D projection\n"
+        + f"[layer {output_layer_id}, dim: {features_dim}]"
+    )
+
+    ax = plt.axes(plt.gca())
+    if n_components == 3:
+        if ax.name != "3d":
+            ax.remove()
+            ax = plt.axes(projection="3d")
+        ax.set_facecolor("white")
+
+    # 2D projection
+    if n_components == 2:
+        # id data
+        x, y = features_proj.T
+        df = pd.DataFrame(
+            {
+                "dim 1": x,
+                "dim 2": y,
+                "Class": labels_str,
+                "Data type": data_type,
+            }
+        )
+        s = sns.scatterplot(
+            data=df,
+            x="dim 1",
+            y="dim 2",
+            hue="Class",
+            hue_order=np.unique(df["Class"]),
+            size="Data type",
+            sizes=[40, 20],
+            style="Data type",
+            size_order=["ID", "OOD"],
+            style_order=["ID", "OOD"],
+            ax=ax,
+        )
+        s.legend(fontsize=8, bbox_to_anchor=(1.1, 1), borderaxespad=0)
+
+    # 3D projection
+    elif n_components == 3:
+        cmap = plt.get_cmap(
+            "tab10", int(np.max(in_labels)) - int(np.min(in_labels)) + 1
+        )
+        # id
+        x_in, y_in, z_in = features_proj[len(out_features) :].T
+        s = ax.scatter(
+            x_in,
+            y_in,
+            z_in,
+            c=in_labels,
+            marker="D",
+            label="ID data",
+            s=30,
+            alpha=1.0,
+            cmap=cmap,
+            vmin=np.min(in_labels) - 0.5,
+            vmax=np.max(in_labels) + 0.5,
+            edgecolors="white",
+            linewidths=0.3,
+        )
+        # ood
+        x_out, y_out, z_out = features_proj[: len(out_features)].T
+        ax.scatter(
+            x_out,
+            y_out,
+            z_out,
+            c="darkslategray",
+            marker="o",
+            label="OOD data",
+            s=15,
+            alpha=1.0,
+            edgecolors="white",
+            linewidths=0.3,
+        )
+        legend_elements = [
+            Line2D(
+                [],
+                [],
+                marker="D",
+                color="white",
+                linestyle="None",
+                label=f"class {v}",
+                markerfacecolor=cmap(v),
+                markersize=7,
+                linewidth=0.3,
+            )
+            for v in np.unique(in_labels)
+        ] + [
+            Line2D(
+                [],
+                [],
+                marker="o",
+                color="white",
+                linestyle="None",
+                label="unknown",
+                markerfacecolor="darkslategray",
+                markersize=7,
+                linewidth=0.3,
+            )
+        ]
+        ax.legend(
+            title="classes",
+            handles=legend_elements,
+            loc="upper right",
+            fontsize=8,
+            bbox_to_anchor=(1.35, 1),
+            borderaxespad=0,
+        )
+
+    plt.title(title, weight="bold").set_fontsize(11)
+    if n_components == 2:
+        ax.set_xlabel("Dimension 1")
+        ax.set_ylabel("Dimension 2")
+    if n_components == 3:
+        ax.set_xlabel("Dim 1")
+        ax.set_ylabel("Dim 2")
+        X = np.concatenate([x_in, x_out])
+        Y = np.concatenate([y_in, y_out])
+        Z = np.concatenate([z_in, z_out])
+        ax.set_xlim([X.min(), X.max()])
+        ax.set_ylim([Y.min(), Y.max()])
+        ax.set_zlim([Z.min(), Z.max()])