supervisely 6.73.214__py3-none-any.whl → 6.73.215__py3-none-any.whl

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/outcome_counts.py

@@ -0,0 +1,336 @@
+from collections import defaultdict
+from typing import List
+
+import numpy as np
+
+from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
+    BaseVisMetric,
+)
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget
+from supervisely.nn.task_type import TaskType
+
+
+class OutcomeCounts(BaseVisMetric):
+    CHART_MAIN = "chart_outcome_counts"
+    CHART_COMPARISON = "chart_outcome_counts_comparison"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.imgIds_to_anns = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+        self.coco_to_sly_ids = defaultdict(lambda: defaultdict(lambda: defaultdict(tuple)))
+
+        self._initialize_ids_mapping()
+
+        self.common_and_diff_tp = self._find_common_and_diff_tp()
+        self.common_and_diff_fn = self._find_common_and_diff_fn()
+        self.common_and_diff_fp = self._find_common_and_diff_fp()
+
+    def _initialize_ids_mapping(self):
+        for idx, r in enumerate(self.eval_results):
+            l = {
+                "TP": (r.mp.cocoDt.anns, r.mp.m.tp_matches, r.click_data.outcome_counts),
+                "FN": (r.mp.cocoGt.anns, r.mp.m.fn_matches, r.click_data.outcome_counts),
+                "FP": (r.mp.cocoDt.anns, r.mp.m.fp_matches, r.click_data.outcome_counts),
+            }
+            for outcome, (coco_anns, matches_data, sly_data) in l.items():
+                for m, sly_m in zip(matches_data, sly_data[outcome]):
+                    key = m["dt_id"] if outcome != "FN" else m["gt_id"]
+                    gt_key = m["gt_id"] if outcome != "FP" else m["dt_id"]
+                    ann = coco_anns[key]
+                    self.imgIds_to_anns[idx][outcome][key].append(ann)
+                    self.coco_to_sly_ids[idx][outcome][gt_key] = sly_m
+
+    @property
+    def chart_widget_main(self) -> ChartWidget:
+        chart = ChartWidget(name=self.CHART_MAIN, figure=self.get_main_figure())
+        chart.set_click_data(
+            gallery_id=self.explore_modal_table.id,
+            click_data=self.get_main_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].y}${'_'}${payload.points[0].data.name}`,",
+        )
+        return chart
+
+    @property
+    def chart_widget_comparison(self) -> ChartWidget:
+        chart = ChartWidget(
+            name=self.CHART_COMPARISON,
+            figure=self.get_comparison_figure(),
+        )
+        chart.set_click_data(
+            gallery_id=self.explore_modal_table.id,
+            click_data=self.get_comparison_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].y}${'_'}${payload.points[0].data.name}`,",
+        )
+        return chart
+
+    def _update_figure_layout(self, fig):
+        fig.update_layout(
+            barmode="stack",
+            width=600,
+            height=300,
+        )
+        fig.update_xaxes(title_text="Count (objects)")
+        fig.update_yaxes(tickangle=-90)
+
+        fig.update_layout(
+            dragmode=False,
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig
+
+    def get_main_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+        tp_counts = [eval_result.mp.TP_count for eval_result in self.eval_results][::-1]
+        fn_counts = [eval_result.mp.FN_count for eval_result in self.eval_results][::-1]
+        fp_counts = [eval_result.mp.FP_count for eval_result in self.eval_results][::-1]
+        model_names = [f"Model {idx}" for idx in range(1, len(self.eval_results) + 1)][::-1]
+        counts = [tp_counts, fn_counts, fp_counts]
+        names = ["TP", "FN", "FP"]
+        colors = ["#8ACAA1", "#dd3f3f", "#F7ADAA"]
+
+        for metric, values, color in zip(names, counts, colors):
+            fig.add_trace(
+                go.Bar(
+                    x=values,
+                    y=model_names,
+                    name=metric,
+                    orientation="h",
+                    marker=dict(color=color),
+                    hovertemplate=f"{metric}: %{{x}} objects<extra></extra>",
+                )
+            )
+
+        fig = self._update_figure_layout(fig)
+        return fig
+
+    def get_comparison_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        colors = ["#8ACAA1", "#dd3f3f", "#F7ADAA"]
+        model_names = [f"Model {idx}" for idx in range(1, len(self.eval_results) + 1)][::-1]
+        model_names.append("Common")
+
+        diff_tps, common_tps = self.common_and_diff_tp
+        diff_fns, common_fns = self.common_and_diff_fn
+        diff_fps, common_fps = self.common_and_diff_fp
+        tps_cnt = [len(x) for x in diff_tps[::-1]] + [len(common_tps)]
+        fns_cnt = [len(x) for x in diff_fns[::-1]] + [len(common_fns)]
+        fps_cnt = [len(x) for x in diff_fps[::-1]] + [len(common_fps)]
+
+        for metric, values, color in zip(["TP", "FN", "FP"], [tps_cnt, fns_cnt, fps_cnt], colors):
+            fig.add_trace(
+                go.Bar(
+                    x=values,
+                    y=model_names,
+                    name=metric,
+                    orientation="h",
+                    marker=dict(color=color),
+                    hovertemplate=f"{metric}: %{{x}} objects<extra></extra>",
+                )
+            )
+
+        fig = self._update_figure_layout(fig)
+        return fig
+
+    def _find_common_and_diff_fn(self) -> List[int]:
+        ids = [
+            dict([(x["gt_obj_id"], x["gt_img_id"]) for x in r.click_data.outcome_counts["FN"]])
+            for r in self.eval_results
+        ]
+        same = set.intersection(*[set(x.keys()) for x in ids])
+        diffs = [set(x.keys()) - same for x in ids]
+
+        same = {i: ids[0][i] for i in same}
+        diffs = [{i: s[i] for i in d} for s, d in zip(ids, diffs)]
+
+        return diffs, same
+
+    def _get_coco_key_name(self):
+        task_type_to_key_name = {
+            TaskType.OBJECT_DETECTION: "bbox",
+            TaskType.INSTANCE_SEGMENTATION: "segmentation",
+            TaskType.SEMANTIC_SEGMENTATION: "segmentation",
+        }
+        key_name = task_type_to_key_name.get(self.eval_results[0].cv_task)
+        if key_name is None:
+            raise NotImplementedError("Not implemented for this task type")
+        return key_name
+
+    def _find_common_and_diff_fp(self) -> List[int]:
+        from pycocotools import mask as maskUtils  # pylint: disable=import-error
+
+        iouThr = 0.75
+        key_name = self._get_coco_key_name()
+
+        imgIds_to_anns = [self.imgIds_to_anns[idx]["FP"] for idx in range(len(self.eval_results))]
+        sly_ids_list = [
+            {x["dt_obj_id"]: x["dt_img_id"] for x in r.click_data.outcome_counts["FP"]}
+            for r in self.eval_results
+        ]
+
+        same_fp_matches = []
+        for img_id in imgIds_to_anns[0]:
+            anns_list = [imgIds[img_id] for imgIds in imgIds_to_anns]
+            geoms_list = [[x[key_name] for x in anns] for anns in anns_list]
+
+            if any(len(geoms) == 0 for geoms in geoms_list):
+                continue
+
+            ious_list = [
+                maskUtils.iou(geoms_list[0], geoms, [0] * len(geoms)) for geoms in geoms_list[1:]
+            ]
+            if any(len(ious) == 0 for ious in ious_list):
+                continue
+
+            indxs_list = [np.nonzero(ious > iouThr) for ious in ious_list]
+            if any(len(indxs[0]) == 0 for indxs in indxs_list):
+                continue
+
+            indxs_list = [list(zip(*indxs)) for indxs in indxs_list]
+            indxs_list = [
+                sorted(indxs, key=lambda x: ious[x[0], x[1]], reverse=True)
+                for indxs, ious in zip(indxs_list, ious_list)
+            ]
+
+            id_sets = [set(idxs[0]) for idxs in indxs_list]
+            common_ids = set.intersection(*id_sets)
+            if not common_ids:
+                continue
+
+            for i, j in indxs_list[0]:
+                if i in common_ids:
+                    same_fp_matches.append((anns_list[0][i], [anns[j] for anns in anns_list[1:]]))
+                    common_ids.remove(i)
+
+        # Find different FP matches for each model
+        same_fp_ids = set(x[0]["id"] for x in same_fp_matches)
+        diff_fp_matches = [
+            set([x["dt_id"] for x in eval_result.mp.m.fp_matches]) - same_fp_ids
+            for eval_result in self.eval_results
+        ]
+
+        diff_fp_matches_dicts = []
+        for idx, diff_fp in enumerate(diff_fp_matches):
+            diff_fp_dict = {}
+            for x in diff_fp:
+                obj_id = self.coco_to_sly_ids[idx]["FP"][x]["dt_obj_id"]
+                img_id = sly_ids_list[idx][obj_id]
+                diff_fp_dict[obj_id] = img_id
+            diff_fp_matches_dicts.append(diff_fp_dict)
+
+        same_fp_matches_dict = {}
+        for x in same_fp_matches:
+            obj_id = self.coco_to_sly_ids[0]["FP"][x[0]["id"]]["dt_obj_id"]
+            img_id = sly_ids_list[0][obj_id]
+            same_fp_matches_dict[obj_id] = img_id
+
+        return diff_fp_matches_dicts, same_fp_matches_dict
+
+    def _find_common_and_diff_tp(self) -> tuple:
+
+        ids = [
+            dict([(x["gt_obj_id"], x) for x in r.click_data.outcome_counts["TP"]])
+            for r in self.eval_results
+        ]
+
+        same = set.intersection(*[set(x.keys()) for x in ids])
+        diffs = [set(x.keys()) - same for x in ids]
+
+        same = {s["dt_obj_id"]: s["dt_img_id"] for s in [ids[0][i] for i in same]}
+        diffs = [{s[i]["dt_obj_id"]: s[i]["dt_img_id"] for i in d} for s, d in zip(ids, diffs)]
+
+        return diffs, same
+
+    def get_main_click_data(self):
+        res = {}
+
+        res["layoutTemplate"] = [None, None, None]
+        res["clickData"] = {}
+        for i, eval_result in enumerate(self.eval_results, 1):
+            model_name = f"Model {i}"
+            for outcome, matches_data in eval_result.click_data.outcome_counts.items():
+                key = f"{model_name}_{outcome}"
+                outcome_dict = res["clickData"].setdefault(key, {})
+                outcome_dict["imagesIds"] = []
+
+                img_ids = set()
+                obj_ids = set()
+                for x in matches_data:
+                    img_ids.add(x["dt_img_id"] if outcome != "FN" else x["gt_img_id"])
+                    obj_ids.add(x["dt_obj_id"] if outcome != "FN" else x["gt_obj_id"])
+
+                title = f"{model_name}. {outcome}: {len(obj_ids)} object{'s' if len(obj_ids) > 1 else ''}"
+                outcome_dict["title"] = title
+                outcome_dict["imagesIds"] = list(img_ids)
+                thr = eval_result.f1_optimal_conf
+                if outcome == "FN":
+                    outcome_dict["filters"] = [
+                        {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
+                    ]
+                else:
+                    outcome_dict["filters"] = [
+                        {"type": "tag", "tagId": "outcome", "value": outcome},
+                        {"type": "tag", "tagId": "confidence", "value": [thr, 1]},
+                    ]
+
+        return res
+
+    def get_comparison_click_data(self):
+        res = {}
+
+        res["layoutTemplate"] = [None, None, None]
+
+        res["clickData"] = {}
+
+        outcomes_ids = {
+            "TP": self.common_and_diff_tp,
+            "FN": self.common_and_diff_fn,
+            "FP": self.common_and_diff_fp,
+        }
+
+        def _update_outcome_dict(title, outcome, outcome_dict, ids):
+            img_ids = set()
+            obj_ids = set()
+            for obj_id, img_id in ids.items():
+                img_ids.add(img_id)
+                obj_ids.add(obj_id)
+
+            title = f"{title}. {outcome}: {len(obj_ids)} object{'s' if len(obj_ids) > 1 else ''}"
+            outcome_dict["title"] = title
+            outcome_dict["imagesIds"] = list(img_ids)
+            filters = outcome_dict.setdefault("filters", [])
+            filters.append({"type": "specific_objects", "tagId": None, "value": list(obj_ids)})
+            if outcome != "FN":
+                filters.append({"type": "tag", "tagId": "confidence", "value": [0, 1]})
+                filters.append({"type": "tag", "tagId": "outcome", "value": outcome})
+
+        for outcome, (diff_ids, common_ids) in outcomes_ids.items():
+            key = f"Common_{outcome}"
+            outcome_dict = res["clickData"].setdefault(key, {})
+
+            _update_outcome_dict("Common", outcome, outcome_dict, common_ids)
+
+            for i, diff_ids in enumerate(diff_ids, 1):
+                name = f"Model {i}"
+                key = f"{name}_{outcome}"
+                outcome_dict = res["clickData"].setdefault(key, {})
+
+                _update_outcome_dict(name, outcome, outcome_dict, diff_ids)
+
+        return res

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/overview.py

@@ -0,0 +1,249 @@
+from typing import List
+
+from supervisely._utils import abs_url
+from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
+from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
+    BaseVisMetric,
+)
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class Overview(BaseVisMetric):
+
+    MARKDOWN_OVERVIEW = "markdown_overview"
+    MARKDOWN_OVERVIEW_INFO = "markdown_overview_info"
+    MARKDOWN_COMMON_OVERVIEW = "markdown_common_overview"
+    CHART = "chart_key_metrics"
+
+    def __init__(self, vis_texts, eval_results: List[EvalResult]) -> None:
+        """
+        Class to create widgets for the overview block
+        overview_widgets property returns list of MarkdownWidget with information about the model
+        chart_widget property returns ChartWidget with Scatterpolar chart of the base metrics with each
+        evaluation result metrics displayed
+        """
+        super().__init__(vis_texts, eval_results)
+
+    @property
+    def overview_md(self) -> List[MarkdownWidget]:
+        info = []
+        model_names = []
+        for eval_result in self.eval_results:
+            model_name = eval_result.inference_info.get("model_name") or "Custom"
+            model_name = model_name.replace("_", "\_")
+            model_names.append(model_name)
+
+            info.append(
+                [
+                    eval_result.gt_project_info.id,
+                    eval_result.gt_project_info.name,
+                    eval_result.inference_info.get("task_type"),
+                ]
+            )
+        if all([model_name == "Custom" for model_name in model_names]):
+            model_name = "Custom models"
+        elif all([model_name == model_names[0] for model_name in model_names]):
+            model_name = model_names[0]
+        else:
+            model_name = " vs. ".join(model_names)
+
+        info = [model_name] + info[0]
+
+        text_template: str = getattr(self.vis_texts, self.MARKDOWN_COMMON_OVERVIEW)
+        return MarkdownWidget(
+            name=self.MARKDOWN_COMMON_OVERVIEW,
+            title="Overview",
+            text=text_template.format(*info),
+        )
+
+    @property
+    def overview_widgets(self) -> List[MarkdownWidget]:
+        self.formats = []
+        for eval_result in self.eval_results:
+
+            url = eval_result.inference_info.get("checkpoint_url")
+            link_text = eval_result.inference_info.get("custom_checkpoint_path")
+            if link_text is None:
+                link_text = url
+            link_text = link_text.replace("_", "\_")
+
+            checkpoint_name = eval_result.inference_info.get("deploy_params", {}).get(
+                "checkpoint_name", ""
+            )
+            model_name = eval_result.inference_info.get("model_name") or "Custom"
+
+            report = eval_result.api.file.get_info_by_path(
+                eval_result.team_id, eval_result.report_path
+            )
+            report_link = abs_url(f"/model-benchmark?id={report.id}")
+
+            formats = [
+                checkpoint_name,
+                model_name.replace("_", "\_"),
+                checkpoint_name.replace("_", "\_"),
+                eval_result.inference_info.get("architecture"),
+                eval_result.inference_info.get("runtime"),
+                url,
+                link_text,
+                report_link,
+            ]
+            self.formats.append(formats)
+
+        text_template: str = getattr(self.vis_texts, self.MARKDOWN_OVERVIEW_INFO)
+        widgets = []
+        for formats in self.formats:
+            md = MarkdownWidget(
+                name=self.MARKDOWN_OVERVIEW_INFO,
+                title="Overview",
+                text=text_template.format(*formats),
+            )
+            md.is_info_block = True
+            widgets.append(md)
+        return widgets
+
+    def get_table_widget(self, latency, fps) -> TableWidget:
+        res = {}
+
+        metric_renames_map = {"f1": "F1-score"}
+
+        columns = ["metrics"] + [f"[{i+1}] {r.name}" for i, r in enumerate(self.eval_results)]
+
+        all_metrics = [eval_result.mp.metric_table() for eval_result in self.eval_results]
+        res["content"] = []
+
+        same_iou_thr = False
+        if len(set([r.mp.iou_threshold for r in self.eval_results])) == 1:
+            if self.eval_results[0].mp.iou_threshold is not None:
+                same_iou_thr = True
+
+        for idx, metric in enumerate(all_metrics[0].keys()):
+            if idx == 3 and not same_iou_thr:
+                continue
+            metric_name = metric_renames_map.get(metric, metric)
+            values = [m[metric] for m in all_metrics]
+            values = [v if v is not None else "―" for v in values]
+            values = [round(v, 2) if isinstance(v, float) else v for v in values]
+            row = [metric_name] + values
+            dct = {"row": row, "id": metric, "items": row}
+            res["content"].append(dct)
+
+        latency_row = ["Latency (ms)"] + latency
+        res["content"].append({"row": latency_row, "id": latency_row[0], "items": latency_row})
+
+        fps_row = ["FPS"] + fps
+        res["content"].append({"row": fps_row, "id": fps_row[0], "items": fps_row})
+
+        columns_options = [{"disableSort": True} for _ in columns]
+
+        res["columns"] = columns
+        res["columnsOptions"] = columns_options
+
+        return TableWidget(
+            name="table_key_metrics",
+            data=res,
+            show_header_controls=False,
+            fix_columns=1,
+            page_size=len(res["content"]),
+        )
+
+    @property
+    def chart_widget(self) -> ChartWidget:
+        return ChartWidget(name=self.CHART, figure=self.get_figure())
+
+    def get_overview_info(self, eval_result: EvalResult):
+        classes_cnt = len(eval_result.classes_whitelist)
+        classes_str = "classes" if classes_cnt > 1 else "class"
+        classes_str = f"{classes_cnt} {classes_str}"
+
+        train_session, images_str = "", ""
+        gt_project_id = eval_result.gt_project_info.id
+        gt_dataset_ids = eval_result.gt_dataset_ids
+        gt_images_ids = eval_result.gt_images_ids
+        train_info = eval_result.train_info
+        if gt_images_ids is not None:
+            val_imgs_cnt = len(gt_images_ids)
+        elif gt_dataset_ids is not None:
+            datasets = eval_result.gt_dataset_infos
+            val_imgs_cnt = sum(ds.items_count for ds in datasets)
+        else:
+            val_imgs_cnt = eval_result.gt_project_info.items_count
+
+        if train_info:
+            train_task_id = train_info.get("app_session_id")
+            if train_task_id:
+                task_info = eval_result.api.task.get_info_by_id(int(train_task_id))
+                app_id = task_info["meta"]["app"]["id"]
+                train_session = f'- **Training dashboard**:  <a href="/apps/{app_id}/sessions/{train_task_id}" target="_blank">open</a>'
+
+            train_imgs_cnt = train_info.get("images_count")
+            images_str = f", {train_imgs_cnt} images in train, {val_imgs_cnt} images in validation"
+
+        if gt_images_ids is not None:
+            images_str += f". Evaluated using subset - {val_imgs_cnt} images"
+        elif gt_dataset_ids is not None:
+            links = [
+                f'<a href="/projects/{gt_project_id}/datasets/{ds.id}" target="_blank">{ds.name}</a>'
+                for ds in datasets
+            ]
+            images_str += (
+                f". Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
+            )
+        else:
+            images_str += f". Evaluated on the whole project ({val_imgs_cnt} images)"
+
+        return classes_str, images_str, train_session
+
+    def get_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Overall Metrics
+        fig = go.Figure()
+        for i, eval_result in enumerate(self.eval_results):
+            name = f"[{i + 1}] {eval_result.name}"
+            base_metrics = eval_result.mp.base_metrics()
+            r = list(base_metrics.values())
+            theta = [eval_result.mp.metric_names[k] for k in base_metrics.keys()]
+            fig.add_trace(
+                go.Scatterpolar(
+                    r=r + [r[0]],
+                    theta=theta + [theta[0]],
+                    # fill="toself",
+                    name=name,
+                    marker=dict(color=eval_result.color),
+                    hovertemplate=name + "<br>%{theta}: %{r:.2f}<extra></extra>",
+                )
+            )
+        fig.update_layout(
+            polar=dict(
+                radialaxis=dict(
+                    range=[0.0, 1.0],
+                    ticks="outside",
+                ),
+                angularaxis=dict(rotation=90, direction="clockwise"),
+            ),
+            dragmode=False,
+            # title="Overall Metrics",
+            # width=700,
+            # height=500,
+            # autosize=False,
+            margin=dict(l=25, r=25, t=25, b=25),
+        )
+        fig.update_layout(
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            )
+        )
+        return fig