supervisely 6.73.253__py3-none-any.whl → 6.73.255__py3-none-any.whl

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/iou_eou.py

@@ -0,0 +1,68 @@
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class IntersectionErrorOverUnion(BaseVisMetrics):
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "intersection_error_over_union",
+            "Intersection & Error Over Union",
+            text=self.vis_texts.markdown_iou,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("intersection_error_over_union", self.get_figure())
+
+    def get_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+        from plotly.subplots import make_subplots  # pylint: disable=import-error
+
+        labels = ["mIoU", "mBoundaryEoU", "mExtentEoU", "mSegmentEoU"]
+
+        length = len(self.eval_results)
+        cols = 3 if length > 2 else 2
+        cols = 4 if length % 4 == 0 else cols
+        rows = length // cols + (1 if length % cols != 0 else 0)
+
+        fig = make_subplots(rows=rows, cols=cols, specs=[[{"type": "domain"}] * cols] * rows)
+
+        annotations = []
+        for idx, eval_result in enumerate(self.eval_results, start=1):
+            col = idx % cols + (cols if idx % cols == 0 else 0)
+            row = idx // cols + (1 if idx % cols != 0 else 0)
+
+            fig.add_trace(
+                go.Pie(
+                    labels=labels,
+                    values=[
+                        eval_result.mp.iou,
+                        eval_result.mp.boundary_eou,
+                        eval_result.mp.extent_eou,
+                        eval_result.mp.segment_eou,
+                    ],
+                    hole=0.5,
+                    textposition="outside",
+                    textinfo="percent+label",
+                    marker=dict(colors=["#8ACAA1", "#FFE4B5", "#F7ADAA", "#dd3f3f"]),
+                ),
+                row=row,
+                col=col,
+            )
+            
+            text = f"[{idx}] {eval_result.name[:7]}"
+            text += "..." if len(eval_result.name) > 7 else ""
+            annotations.append(
+                dict(
+                    text=text,
+                    x=sum(fig.get_subplot(row, col).x) / 2,
+                    y=sum(fig.get_subplot(row, col).y) / 2,
+                    showarrow=False,
+                    xanchor="center",
+                )
+            )
+        fig.update_layout(annotations=annotations)
+
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/overview.py

@@ -0,0 +1,223 @@
+from typing import List
+
+from supervisely._utils import abs_url
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.evaluation_result import EvalResult
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class Overview(BaseVisMetrics):
+
+    MARKDOWN_OVERVIEW = "markdown_overview"
+    MARKDOWN_OVERVIEW_INFO = "markdown_overview_info"
+    MARKDOWN_COMMON_OVERVIEW = "markdown_common_overview"
+    CHART = "chart_key_metrics"
+
+    def __init__(self, vis_texts, eval_results: List[EvalResult]) -> None:
+        super().__init__(vis_texts, eval_results)
+
+    @property
+    def overview_md(self) -> List[MarkdownWidget]:
+        info = []
+        model_names = []
+        for eval_result in self.eval_results:
+            model_name = eval_result.name or "Custom"
+            model_name = model_name.replace("_", "\_")
+            model_names.append(model_name)
+
+            info.append(
+                [
+                    eval_result.gt_project_info.id,
+                    eval_result.gt_project_info.name,
+                    eval_result.inference_info.get("task_type"),
+                ]
+            )
+        if all([model_name == "Custom" for model_name in model_names]):
+            model_name = "Custom models"
+        elif all([model_name == model_names[0] for model_name in model_names]):
+            model_name = model_names[0]
+        else:
+            model_name = " vs. ".join(model_names)
+
+        info = [model_name] + info[0]
+
+        text_template: str = getattr(self.vis_texts, self.MARKDOWN_COMMON_OVERVIEW)
+        return MarkdownWidget(
+            name=self.MARKDOWN_COMMON_OVERVIEW,
+            title="Overview",
+            text=text_template.format(*info),
+        )
+
+    @property
+    def overview_widgets(self) -> List[MarkdownWidget]:
+        all_formats = []
+        for eval_result in self.eval_results:
+
+            url = eval_result.inference_info.get("checkpoint_url")
+            link_text = eval_result.inference_info.get("custom_checkpoint_path")
+            if link_text is None:
+                link_text = url
+            link_text = link_text.replace("_", "\_")
+
+            checkpoint_name = eval_result.checkpoint_name
+            model_name = eval_result.inference_info.get("model_name") or "Custom"
+
+            report = eval_result.api.file.get_info_by_path(self.team_id, eval_result.report_path)
+            report_link = abs_url(f"/model-benchmark?id={report.id}")
+
+            formats = [
+                checkpoint_name,
+                model_name.replace("_", "\_"),
+                checkpoint_name.replace("_", "\_"),
+                eval_result.inference_info.get("architecture"),
+                eval_result.inference_info.get("runtime"),
+                url,
+                link_text,
+                report_link,
+            ]
+            all_formats.append(formats)
+
+        text_template: str = getattr(self.vis_texts, self.MARKDOWN_OVERVIEW_INFO)
+        widgets = []
+        for formats in all_formats:
+            md = MarkdownWidget(
+                name=self.MARKDOWN_OVERVIEW_INFO,
+                title="Overview",
+                text=text_template.format(*formats),
+            )
+            md.is_info_block = True
+            widgets.append(md)
+        return widgets
+
+    def get_table_widget(self, latency, fps) -> TableWidget:
+        res = {}
+
+        columns = ["metrics"] + [f"[{i+1}] {r.name}" for i, r in enumerate(self.eval_results)]
+
+        all_metrics = [eval_result.mp.key_metrics() for eval_result in self.eval_results]
+        res["content"] = []
+
+        for metric in all_metrics[0].keys():
+            values = [m[metric] for m in all_metrics]
+            values = [v if v is not None else "―" for v in values]
+            values = [round(v, 2) if isinstance(v, float) else v for v in values]
+            row = [metric] + values
+            dct = {"row": row, "id": metric, "items": row}
+            res["content"].append(dct)
+
+        latency_row = ["Latency (ms)"] + latency
+        res["content"].append({"row": latency_row, "id": latency_row[0], "items": latency_row})
+
+        fps_row = ["FPS"] + fps
+        res["content"].append({"row": fps_row, "id": fps_row[0], "items": fps_row})
+
+        columns_options = [{"disableSort": True} for _ in columns]
+
+        res["columns"] = columns
+        res["columnsOptions"] = columns_options
+
+        return TableWidget(
+            name="table_key_metrics",
+            data=res,
+            show_header_controls=False,
+            fix_columns=1,
+            page_size=len(res["content"]),
+        )
+
+    @property
+    def chart_widget(self) -> ChartWidget:
+        return ChartWidget(name=self.CHART, figure=self.get_figure())
+
+    def get_overview_info(self, eval_result: EvalResult):
+        classes_cnt = len(eval_result.classes_whitelist)
+        classes_str = "classes" if classes_cnt > 1 else "class"
+        classes_str = f"{classes_cnt} {classes_str}"
+
+        train_session, images_str = "", ""
+        gt_project_id = eval_result.gt_project_info.id
+        gt_dataset_ids = eval_result.gt_dataset_ids
+        gt_images_cnt = eval_result.val_images_cnt
+        train_info = eval_result.train_info
+        total_imgs_cnt = eval_result.gt_project_info.items_count
+        if gt_images_cnt is not None:
+            val_imgs_cnt = gt_images_cnt
+        elif gt_dataset_ids is not None:
+            datasets = eval_result.gt_dataset_infos
+            val_imgs_cnt = sum(ds.items_count for ds in datasets)
+        else:
+            val_imgs_cnt = eval_result.gt_project_info.items_count
+
+        if train_info:
+            train_task_id = train_info.get("app_session_id")
+            if train_task_id:
+                task_info = eval_result.api.task.get_info_by_id(int(train_task_id))
+                app_id = task_info["meta"]["app"]["id"]
+                train_session = f'- **Training dashboard**:  <a href="/apps/{app_id}/sessions/{train_task_id}" target="_blank">open</a>'
+
+            train_imgs_cnt = train_info.get("images_count")
+            images_str = f", {train_imgs_cnt} images in train, {val_imgs_cnt} images in validation"
+
+        if gt_images_cnt is not None:
+            images_str += (
+                f", total {total_imgs_cnt} images. Evaluated using subset - {val_imgs_cnt} images"
+            )
+        elif gt_dataset_ids is not None:
+            links = [
+                f'<a href="/projects/{gt_project_id}/datasets/{ds.id}" target="_blank">{ds.name}</a>'
+                for ds in datasets
+            ]
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
+        else:
+            images_str += f", total {total_imgs_cnt} images. Evaluated on the whole project ({val_imgs_cnt} images)"
+
+        return classes_str, images_str, train_session
+
+    def get_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Overall Metrics
+        fig = go.Figure()
+        for i, eval_result in enumerate(self.eval_results):
+            name = f"[{i + 1}] {eval_result.name}"
+            base_metrics = eval_result.mp.key_metrics().copy()
+            base_metrics["mPixel accuracy"] = round(base_metrics["mPixel accuracy"] * 100, 2)
+            r = list(base_metrics.values())
+            theta = list(base_metrics.keys())
+            fig.add_trace(
+                go.Scatterpolar(
+                    r=r + [r[0]],
+                    theta=theta + [theta[0]],
+                    name=name,
+                    marker=dict(color=eval_result.color),
+                    hovertemplate=name + "<br>%{theta}: %{r:.2f}<extra></extra>",
+                )
+            )
+        fig.update_layout(
+            polar=dict(
+                radialaxis=dict(
+                    range=[0, 105],
+                    ticks="outside",
+                ),
+                angularaxis=dict(rotation=90, direction="clockwise"),
+            ),
+            dragmode=False,
+            height=500,
+            margin=dict(l=25, r=25, t=25, b=25),
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/renormalized_error_ou.py

@@ -0,0 +1,57 @@
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class RenormalizedErrorOverUnion(BaseVisMetrics):
+
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            "renormalized_error_over_union",
+            "Renormalized Error over Union",
+            text=self.vis_texts.markdown_renormalized_error_ou,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget("intersection_error_over_union", self.get_figure())
+
+    def get_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        labels = ["Boundary EoU", "Extent EoU", "Segment EoU"]
+
+        for idx, eval_result in enumerate(self.eval_results, 1):
+            model_name = f"[{idx}] {eval_result.short_name}"
+
+            fig.add_trace(
+                go.Bar(
+                    x=labels,
+                    y=[
+                        eval_result.mp.boundary_renormed_eou,
+                        eval_result.mp.extent_renormed_eou,
+                        eval_result.mp.segment_renormed_eou,
+                    ],
+                    name=model_name,
+                    text=[
+                        eval_result.mp.boundary_renormed_eou,
+                        eval_result.mp.extent_renormed_eou,
+                        eval_result.mp.segment_renormed_eou,
+                    ],
+                    textposition="outside",
+                    marker=dict(color=eval_result.color, line=dict(width=0.7)),
+                    width=0.4,
+                )
+            )
+
+        fig.update_traces(hovertemplate="%{x}: %{y:.2f}<extra></extra>")
+        fig.update_layout(
+            barmode="group",
+            bargap=0.15,
+            bargroupgap=0.05,
+            width=700 if len(self.eval_results) < 4 else 1000,
+        )
+
+        return fig

supervisely/nn/benchmark/comparison/semantic_segmentation/vis_metrics/speedtest.py

@@ -0,0 +1,314 @@
+from typing import List, Union
+
+from supervisely.imaging.color import hex2rgb
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetrics
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class Speedtest(BaseVisMetrics):
+
+    def is_empty(self) -> bool:
+        return not any(eval_result.speedtest_info for eval_result in self.eval_results)
+
+    def multiple_batche_sizes(self) -> bool:
+        for eval_result in self.eval_results:
+            if eval_result.speedtest_info is None:
+                continue
+            if len(eval_result.speedtest_info["speedtest"]) > 1:
+                return True
+        return False
+
+    @property
+    def latency(self) -> List[Union[int, str]]:
+        latency = []
+        for eval_result in self.eval_results:
+            if eval_result.speedtest_info is None:
+                latency.append("N/A")
+            else:
+                added = False
+                for test in eval_result.speedtest_info["speedtest"]:
+                    if test["batch_size"] == 1:
+                        latency.append(round(test["benchmark"]["total"], 2))
+                        added = True
+                        break
+                if not added:
+                    latency.append("N/A")
+        return latency
+
+    @property
+    def fps(self) -> List[Union[int, str]]:
+        fps = []
+        for eval_result in self.eval_results:
+            if eval_result.speedtest_info is None:
+                fps.append("N/A")
+            else:
+                added = False
+                for test in eval_result.speedtest_info["speedtest"]:
+                    if test["batch_size"] == 1:
+                        fps.append(round(1000 / test["benchmark"]["total"], 2))
+                        added = True
+                        break
+                if not added:
+                    fps.append("N/A")
+        return fps
+
+    @property
+    def md_intro(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            name="speedtest_intro",
+            title="Inference Speed",
+            text=self.vis_texts.markdown_speedtest_intro,
+        )
+
+    @property
+    def intro_table(self) -> TableWidget:
+        columns = ["Model", "Device", "Hardware", "Runtime"]
+        columns_options = [{"disableSort": True} for _ in columns]
+        content = []
+        for i, eval_result in enumerate(self.eval_results, 1):
+            name = f"[{i}] {eval_result.name}"
+            if eval_result.speedtest_info is None:
+                row = [name, "N/A", "N/A", "N/A"]
+                dct = {
+                    "row": row,
+                    "id": name,
+                    "items": row,
+                }
+                content.append(dct)
+                continue
+            model_info = eval_result.speedtest_info.get("model_info", {})
+            device = model_info.get("device", "N/A")
+            hardware = model_info.get("hardware", "N/A")
+            runtime = model_info.get("runtime", "N/A")
+            row = [name, device, hardware, runtime]
+            dct = {
+                "row": row,
+                "id": name,
+                "items": row,
+            }
+            content.append(dct)
+
+        data = {
+            "columns": columns,
+            "columnsOptions": columns_options,
+            "content": content,
+        }
+        return TableWidget(
+            name="speedtest_intro_table",
+            data=data,
+            show_header_controls=False,
+            fix_columns=1,
+        )
+
+    @property
+    def inference_time_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_speedtest_overview_ms.format(100)
+        return MarkdownWidget(
+            name="inference_time_md",
+            title="Overview",
+            text=text,
+        )
+
+    @property
+    def fps_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_speedtest_overview_fps.format(100)
+        return MarkdownWidget(
+            name="fps_md",
+            title="FPS Table",
+            text=text,
+        )
+
+    @property
+    def fps_table(self) -> TableWidget:
+        data = {}
+        batch_sizes = set()
+        max_fps = 0
+        for i, eval_result in enumerate(self.eval_results, 1):
+            data[i] = {}
+            if eval_result.speedtest_info is None:
+                continue
+            speedtests = eval_result.speedtest_info["speedtest"]
+            for test in speedtests:
+                batch_size = test["batch_size"]
+                fps = round(1000 / test["benchmark"]["total"] * batch_size)
+                batch_sizes.add(batch_size)
+                max_fps = max(max_fps, fps)
+                data[i][batch_size] = fps
+
+        batch_sizes = sorted(batch_sizes)
+        columns = ["Model"]
+        columns_options = [{"disableSort": True}]
+        for batch_size in batch_sizes:
+            columns.append(f"Batch size {batch_size}")
+            columns_options.append(
+                {
+                    "subtitle": "imgs/sec",
+                    "tooltip": "Frames (images) per second",
+                    "postfix": "fps",
+                    # "maxValue": max_fps,
+                }
+            )
+
+        content = []
+        for i, eval_result in enumerate(self.eval_results, 1):
+            name = f"[{i}] {eval_result.name}"
+            row = [name]
+            for batch_size in batch_sizes:
+                if batch_size in data[i]:
+                    row.append(data[i][batch_size])
+                else:
+                    row.append("―")
+            content.append(
+                {
+                    "row": row,
+                    "id": name,
+                    "items": row,
+                }
+            )
+        data = {
+            "columns": columns,
+            "columnsOptions": columns_options,
+            "content": content,
+        }
+        return TableWidget(
+            name="fps_table",
+            data=data,
+            show_header_controls=False,
+            fix_columns=1,
+        )
+
+    @property
+    def inference_time_table(self) -> TableWidget:
+        data = {}
+        batch_sizes = set()
+        for i, eval_result in enumerate(self.eval_results, 1):
+            data[i] = {}
+            if eval_result.speedtest_info is None:
+                continue
+            speedtests = eval_result.speedtest_info["speedtest"]
+            for test in speedtests:
+                batch_size = test["batch_size"]
+                ms = round(test["benchmark"]["total"], 2)
+                batch_sizes.add(batch_size)
+                data[i][batch_size] = ms
+
+        batch_sizes = sorted(batch_sizes)
+        columns = ["Model"]
+        columns_options = [{"disableSort": True}]
+        for batch_size in batch_sizes:
+            columns.extend([f"Batch size {batch_size}"])
+            columns_options.extend(
+                [
+                    {"subtitle": "ms", "tooltip": "Milliseconds for batch images", "postfix": "ms"},
+                ]
+            )
+
+        content = []
+        for i, eval_result in enumerate(self.eval_results, 1):
+            name = f"[{i}] {eval_result.name}"
+            row = [name]
+            for batch_size in batch_sizes:
+                if batch_size in data[i]:
+                    row.append(data[i][batch_size])
+                else:
+                    row.append("―")
+            content.append(
+                {
+                    "row": row,
+                    "id": name,
+                    "items": row,
+                }
+            )
+
+        data = {
+            "columns": columns,
+            "columnsOptions": columns_options,
+            "content": content,
+        }
+        return TableWidget(
+            name="inference_time_md",
+            data=data,
+            show_header_controls=False,
+            fix_columns=1,
+        )
+
+    @property
+    def batch_inference_md(self):
+        return MarkdownWidget(
+            name="batch_inference",
+            title="Batch Inference",
+            text=self.vis_texts.markdown_batch_inference,
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(name="speed_charts", figure=self.get_figure())
+
+    def get_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+        from plotly.subplots import make_subplots  # pylint: disable=import-error
+
+        fig = make_subplots(cols=2)
+
+        for idx, eval_result in enumerate(self.eval_results, 1):
+            if eval_result.speedtest_info is None:
+                continue
+            temp_res = {}
+            for test in eval_result.speedtest_info["speedtest"]:
+                batch_size = test["batch_size"]
+
+                std = test["benchmark_std"]["total"]
+                ms = test["benchmark"]["total"]
+                fps = round(1000 / test["benchmark"]["total"] * batch_size)
+
+                ms_line = temp_res.setdefault("ms", {})
+                fps_line = temp_res.setdefault("fps", {})
+                ms_std_line = temp_res.setdefault("ms_std", {})
+
+                ms_line[batch_size] = ms
+                fps_line[batch_size] = fps
+                ms_std_line[batch_size] = round(std, 2)
+
+            error_color = "rgba(" + ",".join(map(str, hex2rgb(eval_result.color))) + ", 0.5)"
+            fig.add_trace(
+                go.Scatter(
+                    x=list(temp_res["ms"].keys()),
+                    y=list(temp_res["ms"].values()),
+                    name=f"[{idx}] {eval_result.name} (ms)",
+                    line=dict(color=eval_result.color),
+                    customdata=list(temp_res["ms_std"].values()),
+                    error_y=dict(
+                        type="data",
+                        array=list(temp_res["ms_std"].values()),
+                        visible=True,
+                        color=error_color,
+                    ),
+                    hovertemplate="Batch Size: %{x}<br>Time: %{y:.2f} ms<br> Standard deviation: %{customdata:.2f} ms<extra></extra>",
+                ),
+                col=1,
+                row=1,
+            )
+            fig.add_trace(
+                go.Scatter(
+                    x=list(temp_res["fps"].keys()),
+                    y=list(temp_res["fps"].values()),
+                    name=f"[{idx}] {eval_result.name} (fps)",
+                    line=dict(color=eval_result.color),
+                    hovertemplate="Batch Size: %{x}<br>FPS: %{y:.2f}<extra></extra>",  # <br> Standard deviation: %{customdata:.2f}<extra></extra>",
+                ),
+                col=2,
+                row=1,
+            )
+
+        fig.update_xaxes(title_text="Batch size", col=1, dtick=1)
+        fig.update_xaxes(title_text="Batch size", col=2, dtick=1)
+
+        fig.update_yaxes(title_text="Time (ms)", col=1)
+        fig.update_yaxes(title_text="FPS", col=2)
+        fig.update_layout(height=400)
+
+        return fig