supervisely 6.73.238__py3-none-any.whl → 6.73.240__py3-none-any.whl

supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+from typing import Dict  # , Optional
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class OutcomeCounts(DetectionVisMetric):
+    MARKDOWN = "outcome_counts"
+    CHART = "outcome_counts"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_outcome_counts.format(
+            self.vis_texts.definitions.true_positives,
+            self.vis_texts.definitions.false_positives,
+            self.vis_texts.definitions.false_negatives,
+        )
+        return MarkdownWidget(self.MARKDOWN, "Outcome Counts", text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget(self.CHART, self._get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].data.name}`,",
+        )
+        # self.explore_modal_table.set_click_data(
+        #     self.diff_modal_table.id,
+        #     self.get_diff_data(),
+        #     get_key="(payload) => `${payload.annotation.image_id}`",
+        # )
+        return chart
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+        fig.add_trace(
+            go.Bar(
+                x=[self.eval_result.mp.TP_count],
+                y=["Outcome"],
+                name="TP",
+                orientation="h",
+                marker=dict(color="#8ACAA1"),
+                hovertemplate="TP: %{x} objects<extra></extra>",
+            )
+        )
+        fig.add_trace(
+            go.Bar(
+                x=[self.eval_result.mp.FN_count],
+                y=["Outcome"],
+                name="FN",
+                orientation="h",
+                marker=dict(color="#dd3f3f"),
+                hovertemplate="FN: %{x} objects<extra></extra>",
+            )
+        )
+        fig.add_trace(
+            go.Bar(
+                x=[self.eval_result.mp.FP_count],
+                y=["Outcome"],
+                name="FP",
+                orientation="h",
+                marker=dict(color="#F7ADAA"),
+                hovertemplate="FP: %{x} objects<extra></extra>",
+            )
+        )
+        fig.update_layout(
+            barmode="stack",
+            width=600,
+            height=300,
+        )
+        fig.update_xaxes(title_text="Count (objects)")
+        fig.update_yaxes(tickangle=-90)
+
+        fig.update_layout(
+            dragmode=False,
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig
+
+    def get_click_data(self) -> Dict:
+        if not self.clickable:
+            return
+        res = {}
+
+        res["layoutTemplate"] = [None, None, None]
+        res["clickData"] = {}
+        for outcome, matches_data in self.eval_result.click_data.outcome_counts.items():
+            res["clickData"][outcome] = {}
+            res["clickData"][outcome]["imagesIds"] = []
+
+            img_ids = set()
+            for match_data in matches_data:
+                pairs_data = self.eval_result.matched_pair_data[match_data["gt_img_id"]]
+                if outcome == "FN":
+                    img_ids.add(pairs_data.diff_image_info.id)
+                else:
+                    img_ids.add(pairs_data.pred_image_info.id)
+
+            res["clickData"][outcome][
+                "title"
+            ] = f"{outcome}: {len(matches_data)} object{'s' if len(matches_data) > 1 else ''}"
+            res["clickData"][outcome]["imagesIds"] = list(img_ids)
+            res["clickData"][outcome]["filters"] = [
+                {
+                    "type": "tag",
+                    "tagId": "confidence",
+                    "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                },
+                {"type": "tag", "tagId": "outcome", "value": outcome},
+            ]
+
+        return res
+
+    # def get_diff_data(self) -> Dict:
+    #     res = {}
+
+    #     res["layoutTemplate"] = [
+    #         {"skipObjectTagsFiltering": True, "columnTitle": "Ground Truth"},
+    #         {"skipObjectTagsFiltering": ["outcome"], "columnTitle": "Prediction"},
+    #         {"skipObjectTagsFiltering": ["confidence"], "columnTitle": "Difference"},
+    #     ]
+
+    #     click_data = res.setdefault("clickData", {})
+    #     for outcome, matches_data in self.eval_result.click_data.outcome_counts.items():
+    #         filters = [
+    #             {
+    #                 "type": "tag",
+    #                 "tagId": "confidence",
+    #                 "value": [self.eval_result.mp.f1_optimal_conf, 1],
+    #             },
+    #             {"type": "tag", "tagId": "outcome", "value": outcome},
+    #         ]
+    #         for match_data in matches_data:
+    #             pairs_data = self.eval_result.matched_pair_data[match_data["gt_img_id"]]
+    #             gt = pairs_data.gt_image_info
+    #             pred = pairs_data.pred_image_info
+    #             diff = pairs_data.diff_image_info
+    #             assert gt.name == pred.name == diff.name
+    #             for img_id in [pred.id, diff.id]:
+    #                 key = click_data.setdefault(str(img_id), {})
+    #                 key["imagesIds"] = [gt.id, pred.id, diff.id]
+    #                 key["filters"] = filters
+    #                 key["title"] = f"Image: {gt.name}"
+
+    #                 object_bindings = []
+    #                 for img in [pred, diff]:
+    #                     if img == pred:
+    #                         ann_json = pairs_data.pred_annotation.to_json()
+    #                     else:
+    #                         ann_json = pairs_data.diff_annotation.to_json()
+    #                     for obj in ann_json["objects"]:
+    #                         for tag in obj["tags"]:
+    #                             if tag["name"] == "matched_gt_id":
+    #                                 object_bindings.append(
+    #                                     [
+    #                                         {
+    #                                             "id": obj["id"],
+    #                                             "annotationKey": img.id,
+    #                                         },
+    #                                         {
+    #                                             "id": int(tag["value"]),
+    #                                             "annotationKey": gt.id if img == pred else pred.id,
+    #                                         },
+    #                                     ]
+    #                                 )
+    #                 key["objectsBindings"] = object_bindings
+
+    #     return res

supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts_per_class.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+from typing import Dict, Literal
+
+import numpy as np
+import pandas as pd
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    CollapseWidget,
+    ContainerWidget,
+    MarkdownWidget,
+    RadioGroupWidget,
+)
+
+
+class PerClassOutcomeCounts(DetectionVisMetric):
+    MARKDOWN = "per_class_outcome_counts"
+    MARKDOWN_2 = "per_class_outcome_counts_2"
+    CHART = "per_class_outcome_counts"
+    COLLAPSE_TIP = "per_class_outcome_counts_collapse"
+    RADIO_GROUP = "per_class_outcome_counts_radio_group"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+        self._keypair_sep: str = "-"
+        self.switchable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_class_outcome_counts_1
+        text = text.format(
+            self.vis_texts.definitions.true_positives,
+            self.vis_texts.definitions.false_positives,
+            self.vis_texts.definitions.false_negatives,
+        )
+        return MarkdownWidget(self.MARKDOWN, "Outcome Counts by Class", text)
+
+    @property
+    def md_2(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_class_outcome_counts_2
+        text = text.format(self.vis_texts.definitions.f1_score)
+        return MarkdownWidget(self.MARKDOWN_2, "Outcome Counts by Class", text)
+
+    @property
+    def collapse(self) -> CollapseWidget:
+        md = MarkdownWidget(
+            name=self.COLLAPSE_TIP,
+            title="Normalization",
+            text=self.vis_texts.markdown_normalization,
+        )
+        return CollapseWidget([md])
+
+    @property
+    def chart(self) -> ContainerWidget:
+        return ContainerWidget(
+            [self.radio_group(), self._get_chart("normalized"), self._get_chart("absolute")],
+            self.CHART,
+        )
+
+    def radio_group(self) -> RadioGroupWidget:
+        return RadioGroupWidget(
+            "Normalization",
+            self.RADIO_GROUP,
+            ["normalized", "absolute"],
+        )
+
+    def _get_chart(self, switch_key: Literal["normalized", "absolute"]) -> ChartWidget:
+        chart = ChartWidget(
+            self.CHART,
+            self._get_figure(switch_key),
+            switch_key=switch_key,
+            switchable=self.switchable,
+            radiogroup_id=self.RADIO_GROUP,
+        )
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].label}${'-'}${payload.points[0].data.name}`, 'keySeparator': '-',",
+        )
+        return chart
+
+    def _get_figure(self, switch_key: Literal["normalized", "absolute"]):  # -> go.Figure:
+        import plotly.express as px  # pylint: disable=import-error
+
+        # Per-class Counts
+        iou_thres = 0
+
+        tp = self.eval_result.mp.true_positives[:, iou_thres]
+        fp = self.eval_result.mp.false_positives[:, iou_thres]
+        fn = self.eval_result.mp.false_negatives[:, iou_thres]
+
+        # normalize
+        support = tp + fn
+        with np.errstate(invalid="ignore", divide="ignore"):
+            tp_rel = tp / support
+            fp_rel = fp / support
+            fn_rel = fn / support
+
+            # sort by f1
+            sort_scores = 2 * tp / (2 * tp + fp + fn)
+
+        K = len(self.eval_result.mp.cat_names)
+        sort_indices = np.argsort(sort_scores)
+        cat_names_sorted = [self.eval_result.mp.cat_names[i] for i in sort_indices]
+        tp_rel, fn_rel, fp_rel = tp_rel[sort_indices], fn_rel[sort_indices], fp_rel[sort_indices]
+
+        objects_count = np.concatenate([tp[sort_indices], fn[sort_indices], fp[sort_indices]])
+        data = {
+            "Type": ["TP"] * K + ["FN"] * K + ["FP"] * K,
+            "category": cat_names_sorted * 3,
+        }
+        y_label = ""
+        if switch_key == "normalized":
+            y_label = "Objects Fraction"
+            data["count"] = np.concatenate([tp_rel, fn_rel, fp_rel])
+        elif switch_key == "absolute":
+            y_label = "Objects Count"
+            data["count"] = objects_count
+
+        df = pd.DataFrame(data)
+
+        color_map = {"TP": "#8ACAA1", "FN": "#dd3f3f", "FP": "#F7ADAA"}
+        fig = px.bar(
+            df,
+            x="category",
+            y="count",
+            color="Type",
+            height=500,
+            width=1000,
+            labels={"count": y_label, "category": "Class"},
+            color_discrete_map=color_map,
+        )
+        xaxis_title = fig.layout.xaxis.title.text
+        yaxis_title = fig.layout.yaxis.title.text
+        if switch_key == "normalized":
+
+            fig.update_traces(
+                hovertemplate="Type=%{fullData.name} <br>"
+                + xaxis_title
+                + "=%{x}<br>"
+                + yaxis_title
+                + "=%{y:.2f}<extra></extra>"
+            )
+        elif switch_key == "absolute":
+            fig.update_traces(
+                hovertemplate="Type=%{fullData.name} <br>"
+                + xaxis_title
+                + "=%{x}<br>"
+                + yaxis_title
+                + "=%{y}<extra></extra>",
+            )
+        return fig
+
+    def get_click_data(self) -> Dict:
+        if not self.clickable:
+            return
+        res = {}
+        res["layoutTemplate"] = [None, None, None]
+
+        res["clickData"] = {}
+        for class_name, v1 in self.eval_result.click_data.outcome_counts_by_class.items():
+            for outcome, matches_data in v1.items():
+                key = class_name + self._keypair_sep + outcome
+                res["clickData"][key] = {}
+                res["clickData"][key]["imagesIds"] = []
+                res["clickData"][key][
+                    "title"
+                ] = f"Images with objects of class '{class_name}' and outcome '{outcome}'"
+
+                img_ids = set()
+                for match_data in matches_data:
+                    img_comparison_data = self.eval_result.matched_pair_data[
+                        match_data["gt_img_id"]
+                    ]
+                    if outcome == "FN":
+                        img_ids.add(img_comparison_data.diff_image_info.id)
+                    else:
+                        img_ids.add(img_comparison_data.pred_image_info.id)
+                res["clickData"][key]["imagesIds"] = list(img_ids)
+                res["clickData"][key]["filters"] = [
+                    {
+                        "type": "tag",
+                        "tagId": "confidence",
+                        "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                    },
+                    {"type": "tag", "tagId": "outcome", "value": outcome},
+                ]
+        return res

supervisely/nn/benchmark/object_detection/vis_metrics/overview.py

@@ -0,0 +1,116 @@
+import datetime
+from typing import List
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import MarkdownWidget
+
+
+class Overview(DetectionVisMetric):
+
+    def get_header(self, user_login: str) -> MarkdownWidget:
+        current_date = datetime.datetime.now().strftime("%d %B %Y, %H:%M")
+        header_text = self.vis_texts.markdown_header.format(
+            self.eval_result.name, user_login, current_date
+        )
+        header = MarkdownWidget("markdown_header", "Header", text=header_text)
+        return header
+
+    @property
+    def md(self) -> List[MarkdownWidget]:
+        url = self.eval_result.inference_info.get("checkpoint_url")
+        link_text = self.eval_result.inference_info.get("custom_checkpoint_path")
+        if link_text is None:
+            link_text = url
+        link_text = link_text.replace("_", "\_")
+
+        model_name = self.eval_result.inference_info.get("model_name") or "Custom"
+        checkpoint_name = self.eval_result.inference_info.get("deploy_params", {}).get(
+            "checkpoint_name", ""
+        )
+
+        # Note about validation dataset
+        classes_str, note_about_images, starter_app_info = self._get_overview_info()
+
+        # link to scroll to the optimal confidence section
+        opt_conf_url = self.vis_texts.docs_url + "#f1-optimal-confidence-threshold"
+
+        formats = [
+            model_name.replace("_", "\_"),
+            checkpoint_name.replace("_", "\_"),
+            self.eval_result.inference_info.get("architecture"),
+            self.eval_result.inference_info.get("task_type"),
+            self.eval_result.inference_info.get("runtime"),
+            url,
+            link_text,
+            self.eval_result.gt_project_info.id,
+            self.eval_result.gt_project_info.name,
+            classes_str,
+            note_about_images,
+            starter_app_info,
+            self.eval_result.mp.iou_threshold,
+            round(self.eval_result.mp.f1_optimal_conf, 4),
+            opt_conf_url,
+            self.vis_texts.docs_url,
+        ]
+
+        md = MarkdownWidget(
+            "markdown_overview",
+            "Overview",
+            text=self.vis_texts.markdown_overview.format(*formats),
+        )
+        md.is_info_block = True
+        md.width_fit_content = True
+        return md
+
+    def _get_overview_info(self):
+        classes_cnt = len(self.eval_result.classes_whitelist)
+        classes_str = "classes" if classes_cnt > 1 else "class"
+        classes_str = f"{classes_cnt} {classes_str}"
+
+        evaluator_session, train_session, images_str = None, None, ""
+        gt_project_id = self.eval_result.gt_project_info.id
+        gt_dataset_ids = self.eval_result.gt_dataset_ids
+        gt_images_cnt = self.eval_result.val_images_cnt
+        train_info = self.eval_result.train_info
+        evaluator_app_info = self.eval_result.evaluator_app_info
+        total_imgs_cnt = self.eval_result.gt_project_info.items_count
+        if gt_images_cnt is not None:
+            val_imgs_cnt = gt_images_cnt
+        elif gt_dataset_ids is not None:
+            datasets = self.eval_result.gt_dataset_infos
+            val_imgs_cnt = sum(ds.items_count for ds in datasets)
+        else:
+            val_imgs_cnt = self.eval_result.pred_project_info.items_count
+
+        if train_info:
+            train_task_id = train_info.get("app_session_id")
+            if train_task_id:
+                app_id = self.eval_result.task_info["meta"]["app"]["id"]
+                train_session = f'- **Training dashboard**:  <a href="/apps/{app_id}/sessions/{train_task_id}" target="_blank">open</a>'
+
+            train_imgs_cnt = train_info.get("images_count")
+            images_str = f", {train_imgs_cnt} images in train, {val_imgs_cnt} images in validation"
+
+        if gt_images_cnt is not None:
+            images_str += (
+                f", total {total_imgs_cnt} images. Evaluated using subset - {gt_images_cnt} images"
+            )
+        elif gt_dataset_ids is not None:
+            links = [
+                f'<a href="/projects/{gt_project_id}/datasets/{ds.id}" target="_blank">{ds.name}</a>'
+                for ds in datasets
+            ]
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
+        else:
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the whole project ({val_imgs_cnt} images)"
+
+        if evaluator_app_info:
+            evaluator_task_id = evaluator_app_info.get("id")
+            evaluator_app_id = evaluator_app_info.get("meta", {}).get("app", {}).get("id")
+            evaluator_app_name = evaluator_app_info.get("meta", {}).get("app", {}).get("name")
+            if evaluator_task_id and evaluator_app_id and evaluator_app_name:
+                evaluator_session = f'- **Evaluator app session**:  <a href="/apps/{evaluator_app_id}/sessions/{evaluator_task_id}" target="_blank">open</a>'
+
+        starter_app_info = train_session or evaluator_session or ""
+
+        return classes_str, images_str, starter_app_info

supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import numpy as np
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    CollapseWidget,
+    MarkdownWidget,
+    NotificationWidget,
+)
+
+
+class PRCurve(DetectionVisMetric):
+    MARKDOWN = "pr_curve"
+    NOTIFICATION = "pr_curve"
+    COLLAPSE = "pr_curve"
+    CHART = "pr_curve"
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_pr_curve.format(self.vis_texts.definitions.f1_score)
+        return MarkdownWidget(self.MARKDOWN, "Precision-Recall Curve", text)
+
+    @property
+    def notification(self) -> NotificationWidget:
+        title, _ = self.vis_texts.notification_ap.values()
+        return NotificationWidget(
+            self.NOTIFICATION,
+            title.format(self.eval_result.mp.base_metrics()["mAP"].round(2)),
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(self.CHART, self._get_figure())
+
+    @property
+    def collapse(self) -> CollapseWidget:
+        md1 = MarkdownWidget(
+            "pr_curve",
+            "About Trade-offs between precision and recall",
+            self.vis_texts.markdown_trade_offs,
+        )
+        md2 = MarkdownWidget(
+            "what_is_pr_curve",
+            "How the PR curve is built?",
+            self.vis_texts.markdown_what_is_pr_curve.format(
+                self.vis_texts.definitions.confidence_score,
+                self.vis_texts.definitions.true_positives,
+                self.vis_texts.definitions.false_positives,
+            ),
+        )
+        return CollapseWidget([md1, md2])
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.express as px  # pylint: disable=import-error
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        pr_curve = self.eval_result.mp.pr_curve().copy()
+        pr_curve[pr_curve == -1] = np.nan  # -1 is a placeholder for no GT
+        pr_curve = np.nanmean(pr_curve, axis=-1)
+        fig = px.line(
+            x=self.eval_result.mp.recThrs,
+            y=pr_curve,
+            labels={"x": "Recall", "y": "Precision"},
+            width=600,
+            height=500,
+        )
+        fig.data[0].name = "Model"
+        fig.data[0].showlegend = True
+        fig.update_traces(fill="tozeroy", line=dict(color="#1f77b4"))
+        fig.add_trace(
+            go.Scatter(
+                x=self.eval_result.mp.recThrs,
+                y=[1] * len(self.eval_result.mp.recThrs),
+                name="Perfect",
+                line=dict(color="orange", dash="dash"),
+                showlegend=True,
+            )
+        )
+        fig.add_annotation(
+            text=f"mAP = {self.eval_result.mp.base_metrics()['mAP']:.2f}",
+            xref="paper",
+            yref="paper",
+            x=0.98,
+            y=0.92,
+            showarrow=False,
+            bgcolor="white",
+        )
+        fig.update_traces(hovertemplate="Recall: %{x:.2f}<br>Precision: %{y:.2f}<extra></extra>")
+        fig.update_layout(
+            dragmode=False,
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/pr_curve_by_class.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import pandas as pd
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class PRCurveByClass(DetectionVisMetric):
+    MARKDOWN = "pr_curve_by_class"
+    CHART = "pr_curve_by_class"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_pr_by_class
+        return MarkdownWidget(self.MARKDOWN, "PR Curve by Class", text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget(self.CHART, self._get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].data.legendgroup}`,",
+        )
+        return chart
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.express as px  # pylint: disable=import-error
+
+        df = pd.DataFrame(self.eval_result.mp.pr_curve(), columns=self.eval_result.mp.cat_names)
+
+        fig = px.line(
+            df,
+            x=self.eval_result.mp.recThrs,
+            y=df.columns,
+            labels={"x": "Recall", "value": "Precision", "variable": "Category"},
+            color_discrete_sequence=px.colors.qualitative.Prism,
+            width=800,
+            height=600,
+        )
+
+        fig.update_yaxes(range=[0, 1])
+        fig.update_xaxes(range=[0, 1])
+        return fig