supervisely 6.73.254__py3-none-any.whl → 6.73.256__py3-none-any.whl

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/explore_predictions.py

@@ -0,0 +1,141 @@
+from typing import List, Tuple
+
+from supervisely.annotation.annotation import Annotation
+from supervisely.api.image_api import ImageInfo
+from supervisely.api.module_api import ApiField
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import GalleryWidget, MarkdownWidget
+from supervisely.project.project_meta import ProjectMeta
+
+
+class ExplorePredictions(BaseVisMetrics):
+
+    MARKDOWN = "markdown_explorer"
+    GALLERY_DIFFERENCE = "explore_difference_gallery"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.meta = None
+
+    def _merged_meta(self) -> ProjectMeta:
+        if self.meta is not None:
+            return self.meta
+        self.meta = self.eval_results[0].gt_project_meta
+        for eval_res in self.eval_results:
+            self.meta = self.meta.merge(eval_res.pred_project_meta)
+        return self.meta
+
+    @property
+    def difference_predictions_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_explorer
+        return MarkdownWidget(self.MARKDOWN, "Explore Predictions", text)
+
+    @property
+    def explore_gallery(self) -> GalleryWidget:
+        columns_number = len(self.eval_results) + 1
+        images, annotations = self._get_sample_data()
+        gallery = GalleryWidget(self.GALLERY_DIFFERENCE, columns_number=columns_number)
+        gallery.add_image_left_header("Click to explore more")
+        gallery.set_project_meta(self._merged_meta())
+        gallery.set_images(images, annotations)
+        click_data = self.get_click_data_explore_all()
+        gallery.set_click_data(self.explore_modal_table.id, click_data)
+        gallery.set_show_all_data(self.explore_modal_table.id, click_data)
+        gallery._gallery._update_filters()
+
+        return gallery
+
+    def _get_sample_data(self) -> Tuple[List[ImageInfo], List[Annotation], List[ProjectMeta]]:
+        images = []
+        annotations = []
+        api = self.eval_results[0].api
+        names = None
+        ds_name = None
+        for idx, eval_res in enumerate(self.eval_results):
+            if idx == 0:
+                dataset_info = eval_res.gt_dataset_infos[0]
+                infos = api.image.get_list(dataset_info.id, limit=5, force_metadata_for_links=False)
+                ds_name = dataset_info.name
+                images_ids = [image_info.id for image_info in infos]
+                names = [image_info.name for image_info in infos]
+                images.append(infos)
+                from supervisely.api.api import Api
+
+                api: Api
+                anns = api.annotation.download_batch(
+                    dataset_info.id, images_ids, force_metadata_for_links=False
+                )
+                annotations.append(anns)
+            assert ds_name is not None, "Failed to get GT dataset name for gallery"
+
+            dataset_info = api.dataset.get_info_by_name(eval_res.pred_project_id, ds_name)
+
+            assert names is not None, "Failed to get GT image names for gallery"
+            infos = api.image.get_list(
+                dataset_info.id,
+                filters=[
+                    {ApiField.FIELD: ApiField.NAME, ApiField.OPERATOR: "in", ApiField.VALUE: names}
+                ],
+                force_metadata_for_links=False,
+            )
+            images_ids = [image_info.id for image_info in infos]
+            images.append(infos)
+            anns = api.annotation.download_batch(
+                dataset_info.id, images_ids, force_metadata_for_links=False
+            )
+            annotations.append(anns)
+
+        images = list(i for x in zip(*images) for i in x)
+        annotations = list(i for x in zip(*annotations) for i in x)
+        return images, annotations
+
+    def get_click_data_explore_all(self) -> dict:
+        res = {}
+
+        res["projectMeta"] = self._merged_meta().to_json()
+        res["layoutTemplate"] = [{"columnTitle": "Ground Truth"}]
+        for idx, eval_res in enumerate(self.eval_results, 1):
+            res["layoutTemplate"].append({"columnTitle": f"[{idx}] {eval_res.short_name}"})
+
+        click_data = res.setdefault("clickData", {})
+        explore = click_data.setdefault("explore", {})
+        explore["title"] = "Explore all predictions"
+
+        image_names = set()
+        for eval_res in self.eval_results:
+            eval_res.mp.per_image_metrics["img_names"].apply(image_names.add)
+
+        filters = [{"field": "name", "operator": "in", "value": list(image_names)}]
+
+        images_ids = []
+        api = self.eval_results[0].api
+        names = None
+        ds_names = None
+        for idx, eval_res in enumerate(self.eval_results):
+            if idx == 0:
+                dataset_infos = eval_res.gt_dataset_infos
+                ds_names = [ds.name for ds in dataset_infos]
+                current_images_ids = []
+                current_images_names = []
+                for ds in dataset_infos:
+                    image_infos = api.image.get_list(ds.id, filters, force_metadata_for_links=False)
+                    image_infos = sorted(image_infos, key=lambda x: x.name)
+                    current_images_names.extend([image_info.name for image_info in image_infos])
+                    current_images_ids.extend([image_info.id for image_info in image_infos])
+                images_ids.append(current_images_ids)
+                names = current_images_names
+
+            dataset_infos = api.dataset.get_list(eval_res.pred_project_id)
+            dataset_infos = [ds for ds in dataset_infos if ds.name in ds_names]
+            dataset_infos = sorted(dataset_infos, key=lambda x: ds_names.index(x.name))
+            current_images_infos = []
+            for ds in dataset_infos:
+                image_infos = api.image.get_list(ds.id, filters, force_metadata_for_links=False)
+                image_infos = [image_info for image_info in image_infos if image_info.name in names]
+                current_images_infos.extend(image_infos)
+            current_images_infos = sorted(current_images_infos, key=lambda x: names.index(x.name))
+            images_ids.append([image_info.id for image_info in current_images_infos])
+
+        explore["imagesIds"] = list(i for x in zip(*images_ids) for i in x)
+
+        return res

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/frequently_confused.py

@@ -0,0 +1,71 @@
+from typing import List
+
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.semantic_segmentation.evaluator import (
+    SemanticSegmentationEvalResult,
+)
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class FrequentlyConfused(BaseVisMetrics):
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.eval_results: List[SemanticSegmentationEvalResult]
+        self.clickable = True
+        self._keypair_sep = "-"
+
+    @property
+    def md(self) -> MarkdownWidget:
+        if self.is_empty:
+            text = self.vis_texts.markdown_frequently_confused_empty
+        else:
+            text = self.vis_texts.markdown_frequently_confused
+        return MarkdownWidget("frequently_confused", "Frequently Confused Classes", text=text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("frequently_confused", self.get_figure())
+
+    @property
+    def is_empty(self) -> bool:
+        return all(len(e.mp.frequently_confused[0]) == 0 for e in self.eval_results)
+
+    def get_figure(self):
+        import numpy as np
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        classes = self.eval_results[0].classes_whitelist
+
+        model_cnt = len(self.eval_results)
+        all_models_cmat = np.zeros((model_cnt, len(classes), len(classes)))
+        for model_idx, eval_result in enumerate(self.eval_results):
+            cmat, _ = eval_result.mp.confusion_matrix
+            all_models_cmat[model_idx] = cmat[::-1].copy()
+
+        sum_cmat = all_models_cmat.sum(axis=0)
+        np.fill_diagonal(sum_cmat, 0)
+        sum_cmat_flat = sum_cmat.flatten()
+        sorted_indices = np.argsort(sum_cmat_flat)[::-1]
+        n_pairs = min(10, len(classes) * (len(classes) - 1))
+        sorted_indices = sorted_indices[:n_pairs]
+        rows = sorted_indices // len(classes)
+        cols = sorted_indices % len(classes)
+        labels = [f"{classes[rows[i]]}-{classes[cols[i]]}" for i in range(n_pairs)]
+        for model_idx, eval_result in enumerate(self.eval_results):
+            cmat = all_models_cmat[model_idx]
+            probs = cmat[rows, cols] 
+            probs = probs * 100
+            fig.add_trace(
+                go.Bar(
+                    name=eval_result.name,
+                    x=labels,
+                    y=probs,
+                    hovertemplate="%{x}: %{y:.2f}%<extra></extra>",
+                    marker=dict(color=eval_result.color, line=dict(width=0.7)),
+                )
+            )
+
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/iou_eou.py

@@ -0,0 +1,68 @@
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class IntersectionErrorOverUnion(BaseVisMetrics):
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "intersection_error_over_union",
+            "Intersection & Error Over Union",
+            text=self.vis_texts.markdown_iou,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("intersection_error_over_union", self.get_figure())
+
+    def get_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+        from plotly.subplots import make_subplots  # pylint: disable=import-error
+
+        labels = ["mIoU", "mBoundaryEoU", "mExtentEoU", "mSegmentEoU"]
+
+        length = len(self.eval_results)
+        cols = 3 if length > 2 else 2
+        cols = 4 if length % 4 == 0 else cols
+        rows = length // cols + (1 if length % cols != 0 else 0)
+
+        fig = make_subplots(rows=rows, cols=cols, specs=[[{"type": "domain"}] * cols] * rows)
+
+        annotations = []
+        for idx, eval_result in enumerate(self.eval_results, start=1):
+            col = idx % cols + (cols if idx % cols == 0 else 0)
+            row = idx // cols + (1 if idx % cols != 0 else 0)
+
+            fig.add_trace(
+                go.Pie(
+                    labels=labels,
+                    values=[
+                        eval_result.mp.iou,
+                        eval_result.mp.boundary_eou,
+                        eval_result.mp.extent_eou,
+                        eval_result.mp.segment_eou,
+                    ],
+                    hole=0.5,
+                    textposition="outside",
+                    textinfo="percent+label",
+                    marker=dict(colors=["#8ACAA1", "#FFE4B5", "#F7ADAA", "#dd3f3f"]),
+                ),
+                row=row,
+                col=col,
+            )
+            
+            text = f"[{idx}] {eval_result.name[:7]}"
+            text += "..." if len(eval_result.name) > 7 else ""
+            annotations.append(
+                dict(
+                    text=text,
+                    x=sum(fig.get_subplot(row, col).x) / 2,
+                    y=sum(fig.get_subplot(row, col).y) / 2,
+                    showarrow=False,
+                    xanchor="center",
+                )
+            )
+        fig.update_layout(annotations=annotations)
+
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/overview.py

@@ -0,0 +1,223 @@
+from typing import List
+
+from supervisely._utils import abs_url
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class Overview(BaseVisMetrics):
+
+    MARKDOWN_OVERVIEW = "markdown_overview"
+    MARKDOWN_OVERVIEW_INFO = "markdown_overview_info"
+    MARKDOWN_COMMON_OVERVIEW = "markdown_common_overview"
+    CHART = "chart_key_metrics"
+
+    def __init__(self, vis_texts, eval_results: List[EvalResult]) -> None:
+        super().__init__(vis_texts, eval_results)
+
+    @property
+    def overview_md(self) -> List[MarkdownWidget]:
+        info = []
+        model_names = []
+        for eval_result in self.eval_results:
+            model_name = eval_result.name or "Custom"
+            model_name = model_name.replace("_", "\_")
+            model_names.append(model_name)
+
+            info.append(
+                [
+                    eval_result.gt_project_info.id,
+                    eval_result.gt_project_info.name,
+                    eval_result.inference_info.get("task_type"),
+                ]
+            )
+        if all([model_name == "Custom" for model_name in model_names]):
+            model_name = "Custom models"
+        elif all([model_name == model_names[0] for model_name in model_names]):
+            model_name = model_names[0]
+        else:
+            model_name = " vs. ".join(model_names)
+
+        info = [model_name] + info[0]
+
+        text_template: str = getattr(self.vis_texts, self.MARKDOWN_COMMON_OVERVIEW)
+        return MarkdownWidget(
+            name=self.MARKDOWN_COMMON_OVERVIEW,
+            title="Overview",
+            text=text_template.format(*info),
+        )
+
+    @property
+    def overview_widgets(self) -> List[MarkdownWidget]:
+        all_formats = []
+        for eval_result in self.eval_results:
+
+            url = eval_result.inference_info.get("checkpoint_url")
+            link_text = eval_result.inference_info.get("custom_checkpoint_path")
+            if link_text is None:
+                link_text = url
+            link_text = link_text.replace("_", "\_")
+
+            checkpoint_name = eval_result.checkpoint_name
+            model_name = eval_result.inference_info.get("model_name") or "Custom"
+
+            report = eval_result.api.file.get_info_by_path(self.team_id, eval_result.report_path)
+            report_link = abs_url(f"/model-benchmark?id={report.id}")
+
+            formats = [
+                checkpoint_name,
+                model_name.replace("_", "\_"),
+                checkpoint_name.replace("_", "\_"),
+                eval_result.inference_info.get("architecture"),
+                eval_result.inference_info.get("runtime"),
+                url,
+                link_text,
+                report_link,
+            ]
+            all_formats.append(formats)
+
+        text_template: str = getattr(self.vis_texts, self.MARKDOWN_OVERVIEW_INFO)
+        widgets = []
+        for formats in all_formats:
+            md = MarkdownWidget(
+                name=self.MARKDOWN_OVERVIEW_INFO,
+                title="Overview",
+                text=text_template.format(*formats),
+            )
+            md.is_info_block = True
+            widgets.append(md)
+        return widgets
+
+    def get_table_widget(self, latency, fps) -> TableWidget:
+        res = {}
+
+        columns = ["metrics"] + [f"[{i+1}] {r.name}" for i, r in enumerate(self.eval_results)]
+
+        all_metrics = [eval_result.mp.key_metrics() for eval_result in self.eval_results]
+        res["content"] = []
+
+        for metric in all_metrics[0].keys():
+            values = [m[metric] for m in all_metrics]
+            values = [v if v is not None else "―" for v in values]
+            values = [round(v, 2) if isinstance(v, float) else v for v in values]
+            row = [metric] + values
+            dct = {"row": row, "id": metric, "items": row}
+            res["content"].append(dct)
+
+        latency_row = ["Latency (ms)"] + latency
+        res["content"].append({"row": latency_row, "id": latency_row[0], "items": latency_row})
+
+        fps_row = ["FPS"] + fps
+        res["content"].append({"row": fps_row, "id": fps_row[0], "items": fps_row})
+
+        columns_options = [{"disableSort": True} for _ in columns]
+
+        res["columns"] = columns
+        res["columnsOptions"] = columns_options
+
+        return TableWidget(
+            name="table_key_metrics",
+            data=res,
+            show_header_controls=False,
+            fix_columns=1,
+            page_size=len(res["content"]),
+        )
+
+    @property
+    def chart_widget(self) -> ChartWidget:
+        return ChartWidget(name=self.CHART, figure=self.get_figure())
+
+    def get_overview_info(self, eval_result: EvalResult):
+        classes_cnt = len(eval_result.classes_whitelist)
+        classes_str = "classes" if classes_cnt > 1 else "class"
+        classes_str = f"{classes_cnt} {classes_str}"
+
+        train_session, images_str = "", ""
+        gt_project_id = eval_result.gt_project_info.id
+        gt_dataset_ids = eval_result.gt_dataset_ids
+        gt_images_cnt = eval_result.val_images_cnt
+        train_info = eval_result.train_info
+        total_imgs_cnt = eval_result.gt_project_info.items_count
+        if gt_images_cnt is not None:
+            val_imgs_cnt = gt_images_cnt
+        elif gt_dataset_ids is not None:
+            datasets = eval_result.gt_dataset_infos
+            val_imgs_cnt = sum(ds.items_count for ds in datasets)
+        else:
+            val_imgs_cnt = eval_result.gt_project_info.items_count
+
+        if train_info:
+            train_task_id = train_info.get("app_session_id")
+            if train_task_id:
+                task_info = eval_result.api.task.get_info_by_id(int(train_task_id))
+                app_id = task_info["meta"]["app"]["id"]
+                train_session = f'- **Training dashboard**:  <a href="/apps/{app_id}/sessions/{train_task_id}" target="_blank">open</a>'
+
+            train_imgs_cnt = train_info.get("images_count")
+            images_str = f", {train_imgs_cnt} images in train, {val_imgs_cnt} images in validation"
+
+        if gt_images_cnt is not None:
+            images_str += (
+                f", total {total_imgs_cnt} images. Evaluated using subset - {val_imgs_cnt} images"
+            )
+        elif gt_dataset_ids is not None:
+            links = [
+                f'<a href="/projects/{gt_project_id}/datasets/{ds.id}" target="_blank">{ds.name}</a>'
+                for ds in datasets
+            ]
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
+        else:
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the whole project ({val_imgs_cnt} images)"
+
+        return classes_str, images_str, train_session
+
+    def get_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Overall Metrics
+        fig = go.Figure()
+        for i, eval_result in enumerate(self.eval_results):
+            name = f"[{i + 1}] {eval_result.name}"
+            base_metrics = eval_result.mp.key_metrics().copy()
+            base_metrics["mPixel accuracy"] = round(base_metrics["mPixel accuracy"] * 100, 2)
+            r = list(base_metrics.values())
+            theta = list(base_metrics.keys())
+            fig.add_trace(
+                go.Scatterpolar(
+                    r=r + [r[0]],
+                    theta=theta + [theta[0]],
+                    name=name,
+                    marker=dict(color=eval_result.color),
+                    hovertemplate=name + "<br>%{theta}: %{r:.2f}<extra></extra>",
+                )
+            )
+        fig.update_layout(
+            polar=dict(
+                radialaxis=dict(
+                    range=[0, 105],
+                    ticks="outside",
+                ),
+                angularaxis=dict(rotation=90, direction="clockwise"),
+            ),
+            dragmode=False,
+            height=500,
+            margin=dict(l=25, r=25, t=25, b=25),
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/renormalized_error_ou.py

@@ -0,0 +1,57 @@
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class RenormalizedErrorOverUnion(BaseVisMetrics):
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "renormalized_error_over_union",
+            "Renormalized Error over Union",
+            text=self.vis_texts.markdown_renormalized_error_ou,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("intersection_error_over_union", self.get_figure())
+
+    def get_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        labels = ["Boundary EoU", "Extent EoU", "Segment EoU"]
+
+        for idx, eval_result in enumerate(self.eval_results, 1):
+            model_name = f"[{idx}] {eval_result.short_name}"
+
+            fig.add_trace(
+                go.Bar(
+                    x=labels,
+                    y=[
+                        eval_result.mp.boundary_renormed_eou,
+                        eval_result.mp.extent_renormed_eou,
+                        eval_result.mp.segment_renormed_eou,
+                    ],
+                    name=model_name,
+                    text=[
+                        eval_result.mp.boundary_renormed_eou,
+                        eval_result.mp.extent_renormed_eou,
+                        eval_result.mp.segment_renormed_eou,
+                    ],
+                    textposition="outside",
+                    marker=dict(color=eval_result.color, line=dict(width=0.7)),
+                    width=0.4,
+                )
+            )
+
+        fig.update_traces(hovertemplate="%{x}: %{y:.2f}<extra></extra>")
+        fig.update_layout(
+            barmode="group",
+            bargap=0.15,
+            bargroupgap=0.05,
+            width=700 if len(self.eval_results) < 4 else 1000,
+        )
+
+        return fig