supervisely 6.73.238__py3-none-any.whl → 6.73.239__py3-none-any.whl

supervisely/nn/benchmark/object_detection/vis_metrics/precision.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    NotificationWidget,
+)
+
+
+class Precision(DetectionVisMetric):
+    MARKDOWN = "precision"
+    MARKDOWN_PER_CLASS = "precision_per_class"
+    NOTIFICATION = "precision"
+    CHART = "precision"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_P
+        return MarkdownWidget(self.MARKDOWN, "Precision", text)
+
+    @property
+    def notification(self) -> NotificationWidget:
+        title, desc = self.vis_texts.notification_precision.values()
+        tp_plus_fp = self.eval_result.mp.TP_count + self.eval_result.mp.FP_count
+        return NotificationWidget(
+            self.NOTIFICATION,
+            title.format(self.eval_result.mp.base_metrics()["precision"].round(2)),
+            desc.format(self.eval_result.mp.TP_count, tp_plus_fp),
+        )
+
+    @property
+    def per_class_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_P_perclass.format(self.vis_texts.definitions.f1_score)
+        return MarkdownWidget(self.MARKDOWN_PER_CLASS, "Precision per class", text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget(self.CHART, self._get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].label}`,",
+        )
+        return chart
+
+    def _get_figure(self):  #  -> go.Figure
+        import plotly.express as px  # pylint: disable=import-error
+
+        sorted_by_precision = self.eval_result.mp.per_class_metrics().sort_values(by="precision")
+        fig = px.bar(
+            sorted_by_precision,
+            x="category",
+            y="precision",
+            # title="Per-class Precision (Sorted by F1)",
+            color="precision",
+            range_color=[0, 1],
+            color_continuous_scale="Plasma",
+        )
+        fig.update_traces(hovertemplate="Class: %{x}<br>Precision: %{y:.2f}<extra></extra>")
+        if len(sorted_by_precision) <= 20:
+            fig.update_traces(
+                text=sorted_by_precision.round(2),
+                textposition="outside",
+            )
+        fig.update_xaxes(title_text="Class")
+        fig.update_yaxes(title_text="Precision", range=[0, 1])
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/precision_avg_per_class.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class PerClassAvgPrecision(DetectionVisMetric):
+    MARKDOWN = "per_class_avg_precision"
+    CHART = "per_class_avg_precision"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_class_ap
+        text = text.format(self.vis_texts.definitions.average_precision)
+        return MarkdownWidget(self.MARKDOWN, "Average Precision by Class", text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget(self.CHART, self._get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].theta}`,",
+        )
+        return chart
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.express as px  # pylint: disable=import-error
+
+        # AP per-class
+        ap_per_class = self.eval_result.mp.coco_precision[:, :, :, 0, 2].mean(axis=(0, 1))
+        ap_per_class[ap_per_class == -1] = 0  # -1 is a placeholder for no GT
+        labels = dict(r="Average Precision", theta="Class")
+        fig = px.scatter_polar(
+            r=ap_per_class,
+            theta=self.eval_result.mp.cat_names,
+            # title="Per-class Average Precision (AP)",
+            labels=labels,
+            width=800,
+            height=800,
+            range_r=[0, 1],
+        )
+        fig.update_traces(fill="toself")
+        fig.update_layout(
+            modebar_add=["resetScale"],
+            margin=dict(l=80, r=80, t=0, b=0),
+        )
+        fig.update_traces(
+            hovertemplate=labels["theta"]
+            + ": %{theta}<br>"
+            + labels["r"]
+            + ": %{r:.2f}<br>"
+            + "<extra></extra>"
+        )
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/recall.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    NotificationWidget,
+)
+
+
+class Recall(DetectionVisMetric):
+    MARKDOWN = "recall"
+    MARKDOWN_PER_CLASS = "recall_per_class"
+    NOTIFICATION = "recall"
+    CHART = "recall"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_R
+        return MarkdownWidget(self.MARKDOWN, "Recall", text)
+
+    @property
+    def notification(self) -> NotificationWidget:
+        title, desc = self.vis_texts.notification_recall.values()
+        tp_plus_fn = self.eval_result.mp.TP_count + self.eval_result.mp.FN_count
+        return NotificationWidget(
+            self.NOTIFICATION,
+            title.format(self.eval_result.mp.base_metrics()["recall"].round(2)),
+            desc.format(self.eval_result.mp.TP_count, tp_plus_fn),
+        )
+
+    @property
+    def per_class_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_R_perclass.format(self.vis_texts.definitions.f1_score)
+        return MarkdownWidget(self.MARKDOWN_PER_CLASS, "Recall per class", text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget(self.CHART, self._get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].label}`,",
+        )
+        return chart
+
+    def _get_figure(self):  #  -> go.Figure
+        import plotly.express as px  # pylint: disable=import-error
+
+        sorted_by_f1 = self.eval_result.mp.per_class_metrics().sort_values(by="f1")
+        fig = px.bar(
+            sorted_by_f1,
+            x="category",
+            y="recall",
+            color="recall",
+            range_color=[0, 1],
+            color_continuous_scale="Plasma",
+        )
+        fig.update_traces(hovertemplate="Class: %{x}<br>Recall: %{y:.2f}<extra></extra>")
+        if len(sorted_by_f1) <= 20:
+            fig.update_traces(
+                text=sorted_by_f1["recall"].round(2),
+                textposition="outside",
+            )
+        fig.update_xaxes(title_text="Class")
+        fig.update_yaxes(title_text="Recall", range=[0, 1])
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/recall_vs_precision.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
+
+
+class RecallVsPrecision(DetectionVisMetric):
+    MARKDOWN = "recall_vs_precision"
+    CHART = "recall_vs_precision"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_PR.format(self.vis_texts.definitions.f1_score)
+        return MarkdownWidget(self.MARKDOWN, "Recall vs Precision", text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        chart = ChartWidget(self.CHART, self._get_figure())
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].label}`,",
+        )
+        return chart
+
+    def _get_figure(self):  #  -> Optional[go.Figure]
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        blue_color = "#1f77b4"
+        orange_color = "#ff7f0e"
+        sorted_by_f1 = self.eval_result.mp.per_class_metrics().sort_values(by="f1")
+        fig = go.Figure()
+        fig.add_trace(
+            go.Bar(
+                y=sorted_by_f1["precision"],
+                x=sorted_by_f1["category"],
+                name="Precision",
+                marker=dict(color=blue_color),
+            )
+        )
+        fig.add_trace(
+            go.Bar(
+                y=sorted_by_f1["recall"],
+                x=sorted_by_f1["category"],
+                name="Recall",
+                marker=dict(color=orange_color),
+            )
+        )
+        fig.update_layout(barmode="group")
+        fig.update_xaxes(title_text="Class")
+        fig.update_yaxes(title_text="Value", range=[0, 1])
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/reliability_diagram.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    CollapseWidget,
+    MarkdownWidget,
+    NotificationWidget,
+)
+
+
+class ReliabilityDiagram(DetectionVisMetric):
+    MARKDOWN_CALIBRATION_SCORE = "calibration_score"
+    MARKDOWN_CALIBRATION_SCORE_2 = "calibration_score_2"
+    MARKDOWN_RELIABILITY_DIAGRAM = "reliability_diagram"
+    NOTIFICATION = "reliability_diagram"
+    CHART = "reliability_diagram"
+
+    @property
+    def md_calibration_score(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_calibration_score_1
+        text = text.format(self.vis_texts.definitions.confidence_score)
+        return MarkdownWidget(self.MARKDOWN_CALIBRATION_SCORE, "Calibration Score", text)
+
+    @property
+    def collapse_tip(self) -> CollapseWidget:
+        md = MarkdownWidget(
+            "what_is_calibration",
+            "What is calibration?",
+            self.vis_texts.markdown_what_is_calibration,
+        )
+        return CollapseWidget([md])
+
+    @property
+    def md_calibration_score_2(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            self.MARKDOWN_CALIBRATION_SCORE_2,
+            "",
+            self.vis_texts.markdown_calibration_score_2,
+        )
+
+    @property
+    def md_reliability_diagram(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            self.MARKDOWN_RELIABILITY_DIAGRAM,
+            "Reliability Diagram",
+            self.vis_texts.markdown_reliability_diagram,
+        )
+
+    @property
+    def notification(self) -> NotificationWidget:
+        title, _ = self.vis_texts.notification_ece.values()
+        return NotificationWidget(
+            self.NOTIFICATION,
+            title.format(self.eval_result.mp.m_full.expected_calibration_error().round(4)),
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(self.CHART, self._get_figure())
+
+    @property
+    def collapse(self) -> CollapseWidget:
+        md = MarkdownWidget(
+            "markdown_calibration_curve_interpretation",
+            "How to interpret the Calibration curve",
+            self.vis_texts.markdown_calibration_curve_interpretation,
+        )
+        return CollapseWidget([md])
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Calibration curve (only positive predictions)
+        true_probs, pred_probs = self.eval_result.mp.m_full.calibration_curve()
+
+        fig = go.Figure()
+        fig.add_trace(
+            go.Scatter(
+                x=pred_probs,
+                y=true_probs,
+                mode="lines+markers",
+                name="Calibration plot (Model)",
+                line=dict(color="blue"),
+                marker=dict(color="blue"),
+            )
+        )
+        fig.add_trace(
+            go.Scatter(
+                x=[0, 1],
+                y=[0, 1],
+                mode="lines",
+                name="Perfectly calibrated",
+                line=dict(color="orange", dash="dash"),
+            )
+        )
+
+        fig.update_layout(
+            xaxis_title="Confidence Score",
+            yaxis_title="Fraction of True Positives",
+            legend=dict(x=0.6, y=0.1),
+            xaxis=dict(range=[0, 1]),
+            yaxis=dict(range=[0, 1]),
+            width=700,
+            height=500,
+        )
+        fig.update_traces(
+            hovertemplate="Confidence Score: %{x:.2f}<br>Fraction of True Positives: %{y:.2f}<extra></extra>"
+        )
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/speedtest.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class Speedtest(DetectionVisMetric):
+    MARKDOWN_INTRO = "speedtest_intro"
+    TABLE_MARKDOWN = "speedtest_table"
+    TABLE = "speedtest"
+    CHART_MARKDOWN = "speedtest_chart"
+    CHART = "speedtest"
+
+    def is_empty(self) -> bool:
+        return not self.eval_result.speedtest_info
+
+    @property
+    def num_batche_sizes(self) -> int:
+        if self.is_empty():
+            return 0
+        return len(self.eval_result.speedtest_info.get("speedtest", []))
+
+    @property
+    def intro_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_speedtest_intro
+        text = text.format(
+            self.eval_result.speedtest_info["model_info"]["device"],
+            self.eval_result.speedtest_info["model_info"]["hardware"],
+            self.eval_result.speedtest_info["model_info"]["runtime"],
+        )
+        return MarkdownWidget(name=self.MARKDOWN_INTRO, title="Inference Speed", text=text)
+
+    @property
+    def table_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_speedtest_table
+        text = text.format(self.eval_result.speedtest_info["speedtest"][0]["num_iterations"])
+        return MarkdownWidget(name=self.TABLE_MARKDOWN, title="Speedtest Table", text=text)
+
+    @property
+    def table(self) -> TableWidget:
+        columns = [" ", "Inference time", "FPS"]
+        content = []
+        temp_res = {}
+        max_fps = 0
+        for test in self.eval_result.speedtest_info["speedtest"]:
+            batch_size = test["batch_size"]
+
+            ms = round(test["benchmark"]["total"], 2)
+            fps = round(1000 / test["benchmark"]["total"] * batch_size)
+            row = [f"Batch size {batch_size}", ms, fps]
+            temp_res[batch_size] = row
+            max_fps = max(max_fps, fps)
+
+        # sort by batch size
+        temp_res = dict(sorted(temp_res.items()))
+        for row in temp_res.values():
+            content.append({"row": row, "id": row[0], "items": row})
+
+        columns_options = [
+            {"disableSort": True},  # , "ustomCell": True},
+            {"subtitle": "ms", "tooltip": "Milliseconds for batch images", "postfix": "ms"},
+            {
+                "subtitle": "imgs/sec",
+                "tooltip": "Frames (images) per second",
+                "postfix": "fps",
+                "maxValue": max_fps,
+            },
+        ]
+        data = {"columns": columns, "columnsOptions": columns_options, "content": content}
+        table = TableWidget(name=self.TABLE, data=data)
+        table.main_column = "Batch size"
+        table.fixed_columns = 1
+        table.show_header_controls = False
+        return table
+
+    @property
+    def chart_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_speedtest_chart
+        return MarkdownWidget(name=self.CHART_MARKDOWN, title="Speedtest Chart", text=text)
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(name=self.CHART, figure=self._get_figure())
+
+    def _get_figure(self):  #  -> go.Figure:
+        import plotly.graph_objects as go  # pylint: disable=import-error
+        from plotly.subplots import make_subplots  # pylint: disable=import-error
+
+        fig = make_subplots(cols=2)
+
+        ms_color = "#e377c2"
+        fps_color = "#17becf"
+
+        temp_res = {}
+        for test in self.eval_result.speedtest_info["speedtest"]:
+            batch_size = test["batch_size"]
+
+            std = test["benchmark_std"]["total"]
+            ms = test["benchmark"]["total"]
+            fps = round(1000 / test["benchmark"]["total"] * batch_size)
+
+            ms_line = temp_res.setdefault("ms", {})
+            fps_line = temp_res.setdefault("fps", {})
+            ms_std_line = temp_res.setdefault("ms_std", {})
+
+            ms_line[batch_size] = ms
+            fps_line[batch_size] = fps
+            ms_std_line[batch_size] = round(std, 2)
+
+        fig.add_trace(
+            go.Scatter(
+                x=list(temp_res["ms"].keys()),
+                y=list(temp_res["ms"].values()),
+                name="Inference time (ms)",
+                line=dict(color=ms_color),
+                customdata=list(temp_res["ms_std"].values()),
+                error_y=dict(
+                    type="data",
+                    array=list(temp_res["ms_std"].values()),
+                    visible=True,
+                    color="rgba(227, 119, 194, 0.7)",
+                ),
+                hovertemplate="Batch Size: %{x}<br>Time: %{y:.2f} ms<br> Standard deviation: %{customdata:.2f} ms<extra></extra>",
+            ),
+            col=1,
+            row=1,
+        )
+        fig.add_trace(
+            go.Scatter(
+                x=list(temp_res["fps"].keys()),
+                y=list(temp_res["fps"].values()),
+                name="FPS",
+                line=dict(color=fps_color),
+                hovertemplate="Batch Size: %{x}<br>FPS: %{y:.2f}<extra></extra>",
+            ),
+            col=2,
+            row=1,
+        )
+
+        fig.update_xaxes(title_text="Batch size", col=1, dtick=1)
+        fig.update_xaxes(title_text="Batch size", col=2, dtick=1)
+
+        fig.update_yaxes(title_text="Time (ms)", col=1)
+        fig.update_yaxes(title_text="FPS", col=2)
+        fig.update_layout(height=400)
+
+        return fig