supervisely 6.73.237__py3-none-any.whl → 6.73.239__py3-none-any.whl

supervisely/nn/benchmark/{visualization → object_detection}/vis_metrics/f1_score_at_different_iou.py

@@ -1,46 +1,42 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 import numpy as np
 import pandas as pd
 
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import ChartWidget, MarkdownWidget
 
 
-class F1ScoreAtDifferentIOU(MetricVis):
+class F1ScoreAtDifferentIOU(DetectionVisMetric):
+    MARKDOWN = "f1_score_at_iou"
+    CHART = "f1_score_at_iou"
 
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_f1_at_ious=Widget.Markdown(
-                title="Confidence Profile at Different IoU thresholds",
-                is_header=True,
-                formats=[self._loader.vis_texts.definitions.iou_threshold],
-            ),
-            chart=Widget.Chart(),
+    @property
+    def md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            self.MARKDOWN,
+            "Confidence Profile at Different IoU thresholds",
+            self.vis_texts.markdown_f1_at_ious,
         )
 
-    def get_figure(self, widget: Widget):  # -> Optional[go.Figure]:
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(self.CHART, self._get_figure())
+
+    def _get_figure(self):  # -> go.Figure:
         import plotly.express as px  # pylint: disable=import-error
 
-        # score_profile = self._loader.m_full.confidence_score_profile()
-        f1s = self._loader.mp.m_full.score_profile_f1s
+        f1s = self.eval_result.mp.m_full.score_profile_f1s
 
         # downsample
-        if len(self._loader.df_score_profile) > 5000:
+        if len(self.eval_result.df_score_profile) > 5000:
             f1s_down = f1s[:, :: f1s.shape[1] // 1000]
         else:
             f1s_down = f1s
 
-        iou_names = list(map(lambda x: str(round(x, 2)), self._loader.mp.iouThrs.tolist()))
+        iou_names = list(map(lambda x: str(round(x, 2)), self.eval_result.mp.iouThrs.tolist()))
         df = pd.DataFrame(
-            np.concatenate([self._loader.dfsp_down["scores"].values[:, None], f1s_down.T], 1),
+            np.concatenate([self.eval_result.dfsp_down["scores"].values[:, None], f1s_down.T], 1),
             columns=["scores"] + iou_names,
         )
         labels = {"value": "Value", "variable": "IoU threshold", "scores": "Confidence Score"}
@@ -49,7 +45,6 @@ class F1ScoreAtDifferentIOU(MetricVis):
             df,
             x="scores",
             y=iou_names,
-            # title="F1-Score at different IoU Thresholds",
             labels=labels,
             color_discrete_sequence=px.colors.sequential.Viridis,
             width=None,
@@ -64,7 +59,7 @@ class F1ScoreAtDifferentIOU(MetricVis):
         for i, iou in enumerate(iou_names):
             argmax_f1 = f1s[i].argmax()
             max_f1 = f1s[i][argmax_f1]
-            score = self._loader.mp.m_full.score_profile["scores"][argmax_f1]
+            score = self.eval_result.mp.m_full.score_profile["scores"][argmax_f1]
             fig.add_annotation(
                 x=score,
                 y=max_f1,

supervisely/nn/benchmark/object_detection/vis_metrics/frequently_confused.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+from typing import Dict, Literal
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    ContainerWidget,
+    MarkdownWidget,
+    RadioGroupWidget,
+)
+
+
+class FrequentlyConfused(DetectionVisMetric):
+    MARKDOWN = "frequently_confused"
+    MARKDOWN_EMPTY = "frequently_confused_empty"
+    CHART = "frequently_confused"
+    RADIO_GROUP = "frequently_confused_radio_group"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.clickable = True
+        self.df = self.eval_result.mp.frequently_confused()
+        self._keypair_sep: str = "-"
+        self.is_empty = self.df.empty
+        self.switchable = True
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_frequently_confused
+        pair = self.df["category_pair"][0]
+        prob = self.df["probability"][0]
+        text = text.format(
+            pair[0],
+            pair[1],
+            prob.round(2),
+            pair[0],
+            pair[1],
+            (prob * 100).round(),
+            pair[0],
+            pair[1],
+            pair[1],
+            pair[0],
+        )
+        return MarkdownWidget(self.MARKDOWN, "Frequently Confused Classes", text)
+
+    @property
+    def chart(self) -> ContainerWidget:
+        return ContainerWidget(
+            [self.radio_group(), self._get_chart("probability"), self._get_chart("count")],
+            self.CHART,
+        )
+
+    def radio_group(self) -> RadioGroupWidget:
+        return RadioGroupWidget(
+            "Probability or Count",
+            self.RADIO_GROUP,
+            ["probability", "count"],
+        )
+
+    def _get_chart(self, switch_key: Literal["probability", "count"]) -> ChartWidget:
+        chart = ChartWidget(
+            self.CHART,
+            self._get_figure(switch_key),
+            switch_key=switch_key,
+            switchable=self.switchable,
+            radiogroup_id=self.RADIO_GROUP,
+        )
+        chart.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+            chart_click_extra="'getKey': (payload) => `${payload.points[0].x}`, 'keySeparator': '-',",
+        )
+        return chart
+
+    @property
+    def empty_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_frequently_confused_empty
+        return MarkdownWidget(self.MARKDOWN_EMPTY, "Frequently Confused Classes", text)
+
+    def _get_figure(self, switch_key: Literal["probability", "count"]):  # -> go.Figure:
+        if self.is_empty:
+            return
+
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Frequency of confusion as bar chart
+        confused_df = self.eval_result.mp.frequently_confused()
+        confused_name_pairs = confused_df["category_pair"]
+        x_labels = [f"{pair[0]} - {pair[1]}" for pair in confused_name_pairs]
+        y_labels = confused_df[switch_key]
+
+        fig = go.Figure()
+        fig.add_trace(
+            go.Bar(x=x_labels, y=y_labels, marker=dict(color=y_labels, colorscale="Reds"))
+        )
+        fig.update_layout(
+            # title="Frequently confused class pairs",
+            xaxis_title="Class Pair",
+            yaxis_title=y_labels.name.capitalize(),
+            width=1000 if len(x_labels) > 10 else 600,
+        )
+        fig.update_traces(text=y_labels.round(2))
+        fig.update_traces(
+            hovertemplate="Class Pair: %{x}<br>"
+            + y_labels.name.capitalize()
+            + ": %{y:.2f}<extra></extra>"
+        )
+        return fig
+
+    def get_click_data(self) -> Dict:
+        if not self.clickable or self.is_empty:
+            return
+        res = dict(projectMeta=self.eval_result.pred_project_meta.to_json())
+
+        res["layoutTemplate"] = [None, None, None]
+        res["clickData"] = {}
+
+        for keypair, v in self.eval_result.click_data.frequently_confused.items():
+            subkey1, subkey2 = keypair
+            key = f"{subkey1} {self._keypair_sep} {subkey2}"
+            res["clickData"][key] = {}
+            res["clickData"][key]["imagesIds"] = []
+            res["clickData"][key]["title"] = f"Confused classes: {subkey1} - {subkey2}"
+
+            img_ids = set()
+            obj_ids = set()
+            for x in v:
+                img_ids.add(x["dt_img_id"])
+                obj_ids.add(x["dt_obj_id"])
+
+            res["clickData"][key]["imagesIds"] = list(img_ids)
+            res["clickData"][key]["filters"] = [
+                {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
+            ]
+
+        return res

supervisely/nn/benchmark/object_detection/vis_metrics/iou_distribution.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    CollapseWidget,
+    MarkdownWidget,
+    NotificationWidget,
+)
+
+
+class IOUDistribution(DetectionVisMetric):
+    MARKDOWN_LOCALIZATION_ACCURACY = "localization_accuracy"
+    MARKDOWN_IOU_DISTRIBUTION = "iou_distribution"
+    NOTIFICATION = "iou_distribution"
+    COLLAPSE = "iou_distribution"
+    CHART = "iou_distribution"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.md_title = "Localization Accuracy (IoU)"
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_localization_accuracy
+        text = text.format(self.vis_texts.definitions.iou_score)
+        return MarkdownWidget(self.MARKDOWN_LOCALIZATION_ACCURACY, self.md_title, text)
+
+    @property
+    def md_iou_distribution(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_iou_distribution
+        text = text.format(self.vis_texts.definitions.iou_score)
+        return MarkdownWidget(self.MARKDOWN_IOU_DISTRIBUTION, self.md_title, text)
+
+    @property
+    def notification(self) -> NotificationWidget:
+        title, _ = self.vis_texts.notification_avg_iou.values()
+        return NotificationWidget(
+            self.NOTIFICATION,
+            title.format(self.eval_result.mp.base_metrics()["iou"].round(2)),
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(self.CHART, self._get_figure())
+
+    @property
+    def collapse(self) -> CollapseWidget:
+        md1 = MarkdownWidget(
+            "iou_calculation",
+            "How IoU is calculated?",
+            self.vis_texts.markdown_iou_calculation,
+        )
+        md2 = MarkdownWidget(
+            "what_is_pr_curve",
+            "How the PR curve is built?",
+            self.vis_texts.markdown_what_is_pr_curve.format(
+                self.vis_texts.definitions.confidence_score,
+                self.vis_texts.definitions.true_positives,
+                self.vis_texts.definitions.false_positives,
+            ),
+        )
+        return CollapseWidget([md1, md2])
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+        nbins = 40
+        fig.add_trace(go.Histogram(x=self.eval_result.mp.ious, nbinsx=nbins))
+        fig.update_layout(
+            xaxis_title="IoU",
+            yaxis_title="Count",
+            width=600,
+            height=500,
+        )
+
+        # Add annotation for mean IoU as vertical line
+        mean_iou = self.eval_result.mp.ious.mean()
+        y1 = len(self.eval_result.mp.ious) // nbins
+        fig.add_shape(
+            type="line",
+            x0=mean_iou,
+            x1=mean_iou,
+            y0=0,
+            y1=y1,
+            line=dict(color="orange", width=2, dash="dash"),
+        )
+        fig.update_traces(hovertemplate="IoU: %{x:.2f}<br>Count: %{y}<extra></extra>")
+        fig.add_annotation(x=mean_iou, y=y1, text=f"Mean IoU: {mean_iou:.2f}", showarrow=False)
+        fig.update_layout(
+            dragmode=False,
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig

supervisely/nn/benchmark/object_detection/vis_metrics/key_metrics.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from typing import Dict
+
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class KeyMetrics(DetectionVisMetric):
+    MARKDOWN = "key_metrics"
+    CHART = "key_metrics"
+    TABLE = "key_metrics"
+
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_key_metrics.format(
+            self.vis_texts.definitions.iou_threshold,
+            self.vis_texts.definitions.average_precision,
+            self.vis_texts.definitions.confidence_score,
+        )
+        return MarkdownWidget(self.MARKDOWN, "Key Metrics", text)
+
+    @property
+    def table(self) -> TableWidget:
+        columns = ["metrics", "values"]
+        content = []
+        for metric, value in self.eval_result.mp.metric_table().items():
+            row = [metric, round(value, 2)]
+            dct = {
+                "row": row,
+                "id": metric,
+                "items": row,
+            }
+            content.append(dct)
+
+        columns_options = [
+            {"disableSort": True},  # , "ustomCell": True},
+            {"disableSort": True},
+        ]
+
+        data = {
+            "columns": columns,
+            "columnsOptions": columns_options,
+            "content": content,
+        }
+        table = TableWidget(
+            name=self.TABLE,
+            data=data,
+            fix_columns=1,
+            width="60%",
+            show_header_controls=False,
+            main_column=columns[0],
+        )
+        return table
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(self.CHART, self._get_figure())
+
+    def _get_figure(self):  # -> go.Figure:
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Overall Metrics
+        base_metrics = self.eval_result.mp.base_metrics()
+        r = list(base_metrics.values())
+        theta = [self.eval_result.mp.metric_names[k] for k in base_metrics.keys()]
+        fig = go.Figure()
+        fig.add_trace(
+            go.Scatterpolar(
+                r=r + [r[0]],
+                theta=theta + [theta[0]],
+                # fill="toself",
+                name="Overall Metrics",
+                hovertemplate="%{theta}: %{r:.2f}<extra></extra>",
+            )
+        )
+        fig.update_layout(
+            polar=dict(
+                radialaxis=dict(
+                    range=[0.0, 1.0],
+                    ticks="outside",
+                ),
+                angularaxis=dict(rotation=90, direction="clockwise"),
+            ),
+            dragmode=False,
+            margin=dict(l=25, r=25, t=25, b=25),
+        )
+        fig.update_layout(
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            )
+        )
+        return fig
+
+    def get_click_data(self) -> Dict:
+        if not self.clickable:
+            return
+        res = {}
+
+        res["layoutTemplate"] = [None, None, None]
+        res["clickData"] = {}
+        for outcome, matches_data in self.eval_result.click_data.outcome_counts.items():
+            res["clickData"][outcome] = {}
+            res["clickData"][outcome]["imagesIds"] = []
+
+            img_ids = set()
+            for match_data in matches_data:
+                pairs_data = self.eval_result.matched_pair_data[match_data["gt_img_id"]]
+                if outcome == "FN":
+                    img_ids.add(pairs_data.diff_image_info.id)
+                else:
+                    img_ids.add(pairs_data.pred_image_info.id)
+
+            res["clickData"][outcome][
+                "title"
+            ] = f"{outcome}: {len(matches_data)} object{'s' if len(matches_data) > 1 else ''}"
+            res["clickData"][outcome]["imagesIds"] = list(img_ids)
+            res["clickData"][outcome]["filters"] = [
+                {"type": "tag", "tagId": "confidence", "value": [0, 1]},
+                {"type": "tag", "tagId": "outcome", "value": outcome},
+            ]
+
+        return res

supervisely/nn/benchmark/{visualization → object_detection}/vis_metrics/model_predictions.py

@@ -1,42 +1,30 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from typing import Dict
 
-from supervisely.api.image_api import ImageInfo
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
+from supervisely.api.image_api import ImageApi, ImageInfo
+from supervisely.nn.benchmark.object_detection.base_vis_metric import DetectionVisMetric
+from supervisely.nn.benchmark.visualization.widgets import MarkdownWidget, TableWidget
 
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
 
+class ModelPredictions(DetectionVisMetric):
+    MARKDOWN = "model_predictions"
+    TABLE = "model_predictions"
 
-class ModelPredictions(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
         self.clickable = True
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_predictions_gallery=Widget.Markdown(
-                title="Model Predictions", is_header=False
-            ),
-            markdown_predictions_table=Widget.Markdown(
-                title="Prediction details for every image", is_header=True
-            ),
-            # gallery=Widget.Gallery(is_table_gallery=True),
-            table=Widget.Table(),
-        )
-        self._row_ids = None
+        self._row_ids = None  #  TODO: check if this is used
 
-    def get_table(self, widget: Widget.Table) -> dict:
-        res = {}
-        dt_project_id = self._loader.dt_project_info.id
+    @property
+    def md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_predictions_table
+        return MarkdownWidget(self.MARKDOWN, "Prediction details for every image", text)
 
-        tmp = set()
-        for dt_dataset in self._loader._api.dataset.get_list(dt_project_id, recursive=True):
-            names = [x.name for x in self._loader._api.image.get_list(dt_dataset.id)]
-            tmp.update(names)
-        df = self._loader.mp.prediction_table().round(2)
+    @property
+    def table(self) -> TableWidget:
+        tmp = set([d.pred_image_info.name for d in self.eval_result.matched_pair_data.values()])
+        df = self.eval_result.mp.prediction_table().round(2)
         df = df[df["Image name"].isin(tmp)]
         columns_options = [
             {"maxWidth": "225px"},
@@ -58,22 +46,20 @@ class ModelPredictions(MetricVis):
             {"maxValue": 1, "tooltip": "Recall (sensitivity)"},
             {"maxValue": 1, "tooltip": "F1 score (harmonic mean of precision and recall)"},
         ]
-        table_model_preds = widget.table(df, columns_options=columns_options)
-        tbl = table_model_preds.to_json()
 
-        res["columns"] = tbl["columns"][1:]  # exclude sly_id
-        res["columnsOptions"] = columns_options
-        res["content"] = []
+        columns = df.columns.tolist()[1:]  # exclude sly_id
+        content = []
 
         key_mapping = {}
-        for old, new in zip(ImageInfo._fields, self._loader._api.image.info_sequence()):
+        for old, new in zip(ImageInfo._fields, ImageApi.info_sequence()):
             key_mapping[old] = new
 
         self._row_ids = []
+        df = df.replace({float("nan"): None}) # replace NaN / float("nan") with None
 
-        for row in tbl["data"]:
+        for row in df.values.tolist():
             sly_id = row.pop(0)
-            info = self._loader.comparison_data[sly_id].gt_image_info
+            info = self.eval_result.matched_pair_data[sly_id].gt_image_info
 
             dct = {
                 "row": {key_mapping[k]: v for k, v in info._asdict().items()},
@@ -82,11 +68,25 @@ class ModelPredictions(MetricVis):
             }
 
             self._row_ids.append(dct["id"])
-            res["content"].append(dct)
-
-        return res
+            content.append(dct)
+
+        data = {
+            "columns": columns,
+            "columnsOptions": columns_options,
+            "content": content,
+        }
+        table = TableWidget(
+            name=self.TABLE,
+            data=data,
+            fix_columns=1,
+        )
+        table.set_click_data(
+            self.explore_modal_table.id,
+            self.get_click_data(),
+        )
+        return table
 
-    def get_table_click_data(self, widget: Widget.Table) -> Optional[dict]:
+    def get_click_data(self) -> Dict:
         res = {}
         res["layoutTemplate"] = [
             {"skipObjectTagsFiltering": True, "columnTitle": "Ground Truth"},
@@ -96,21 +96,25 @@ class ModelPredictions(MetricVis):
         click_data = res.setdefault("clickData", {})
 
         default_filters = [
-            {"type": "tag", "tagId": "confidence", "value": [self.f1_optimal_conf, 1]},
+            {
+                "type": "tag",
+                "tagId": "confidence",
+                "value": [self.eval_result.mp.f1_optimal_conf, 1],
+            },
             # {"type": "tag", "tagId": "outcome", "value": "FP"},
         ]
 
-        for img_comparison_data in self._loader.comparison_data.values():
-            gt = img_comparison_data.gt_image_info
-            pred = img_comparison_data.pred_image_info
-            diff = img_comparison_data.diff_image_info
+        for pairs_data in self.eval_result.matched_pair_data.values():
+            gt = pairs_data.gt_image_info
+            pred = pairs_data.pred_image_info
+            diff = pairs_data.diff_image_info
             assert gt.name == pred.name == diff.name
             key = click_data.setdefault(str(pred.name), {})
             key["imagesIds"] = [gt.id, pred.id, diff.id]
             key["filters"] = default_filters
             key["title"] = f"Image: {pred.name}"
             image_id = pred.id
-            ann_json = img_comparison_data.pred_annotation.to_json()
+            ann_json = pairs_data.pred_annotation.to_json()
             assert image_id == pred.id
             object_bindings = []
             for obj in ann_json["objects"]:
@@ -130,7 +134,7 @@ class ModelPredictions(MetricVis):
                         )
 
             image_id = diff.id
-            ann_json = img_comparison_data.diff_annotation.to_json()
+            ann_json = pairs_data.diff_annotation.to_json()
             assert image_id == diff.id
             for obj in ann_json["objects"]:
                 for tag in obj["tags"]: