supervisely 6.73.238__py3-none-any.whl → 6.73.240__py3-none-any.whl

supervisely/nn/benchmark/visualization/vis_metrics/classwise_error_analysis.py

@@ -1,55 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List
-
-import pandas as pd
-
-from supervisely.nn.benchmark.cv_tasks import CVTask
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class ClasswiseErrorAnalysis(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.cv_tasks: List[CVTask] = [CVTask.SEMANTIC_SEGMENTATION.value]
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[go.Figure]:
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        pd.options.mode.chained_assignment = None  # TODO rm later
-
-        df = self._loader.result_df
-        df.drop(["mean"], inplace=True)
-        df = df[["IoU", "E_extent_oU", "E_boundary_oU", "E_segment_oU"]]
-        df.sort_values(by="IoU", ascending=False, inplace=True)
-        labels = list(df.index)
-        color_palette = ["cornflowerblue", "moccasin", "lightgreen", "orangered"]
-
-        fig = go.Figure()
-        for i, column in enumerate(df.columns):
-            fig.add_trace(
-                go.Bar(
-                    name=column,
-                    y=df[column],
-                    x=labels,
-                    marker_color=color_palette[i],
-                )
-            )
-        fig.update_yaxes(range=[0, 1])
-        fig.update_layout(
-            barmode="stack",
-            plot_bgcolor="rgba(0, 0, 0, 0)",
-            title={
-                "text": "Classwise segmentation error analysis",
-                "y": 0.9,
-                "x": 0.5,
-                "xanchor": "center",
-                "yanchor": "top",
-            },
-        )
-        return fig

supervisely/nn/benchmark/visualization/vis_metrics/confidence_score.py

@@ -1,93 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class ConfidenceScore(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_confidence_score_1=Widget.Markdown(
-                title="Confidence Score Profile",
-                is_header=True,
-                formats=[self._loader.vis_texts.definitions.confidence_threshold],
-            ),
-            notification_f1=Widget.Notification(
-                formats_title=[round((self._loader.mp.m_full.get_f1_optimal_conf()[0] or 0.0), 4)]
-            ),
-            chart=Widget.Chart(),
-            markdown_confidence_score_2=Widget.Markdown(),
-            collapse_conf_score=Widget.Collapse(
-                Schema(
-                    self._loader.vis_texts,
-                    markdown_plot_confidence_profile=Widget.Markdown(
-                        title="How to plot Confidence Profile?"
-                    ),
-                )
-            ),
-            markdown_calibration_score_3=Widget.Markdown(),
-        )
-
-    def get_figure(self, widget: Widget):  # -> Optional[go.Figure]:
-        import plotly.express as px  # pylint: disable=import-error
-
-        color_map = {
-            "Precision": "#1f77b4",
-            "Recall": "orange",
-        }
-
-        fig = px.line(
-            self._loader.dfsp_down,
-            x="scores",
-            y=["precision", "recall", "f1"],
-            # title="Confidence Score Profile",
-            labels={"value": "Value", "variable": "Metric", "scores": "Confidence Score"},
-            width=None,
-            height=500,
-            color_discrete_map=color_map,
-        )
-        fig.update_traces(
-            hovertemplate="Confidence Score: %{x:.2f}<br>Value: %{y:.2f}<extra></extra>"
-        )
-        fig.update_layout(yaxis=dict(range=[0, 1]), xaxis=dict(range=[0, 1], tick0=0, dtick=0.1))
-
-        if self._loader.mp.f1_optimal_conf is not None and self._loader.mp.best_f1 is not None:
-            # Add vertical line for the best threshold
-            fig.add_shape(
-                type="line",
-                x0=self._loader.mp.f1_optimal_conf,
-                x1=self._loader.mp.f1_optimal_conf,
-                y0=0,
-                y1=self._loader.mp.best_f1,
-                line=dict(color="gray", width=2, dash="dash"),
-            )
-            fig.add_annotation(
-                x=self._loader.mp.f1_optimal_conf,
-                y=self._loader.mp.best_f1 + 0.04,
-                text=f"F1-optimal threshold: {self._loader.mp.f1_optimal_conf:.2f}",
-                showarrow=False,
-            )
-        fig.update_layout(
-            dragmode=False,
-            modebar=dict(
-                remove=[
-                    "zoom2d",
-                    "pan2d",
-                    "select2d",
-                    "lasso2d",
-                    "zoomIn2d",
-                    "zoomOut2d",
-                    "autoScale2d",
-                    "resetScale2d",
-                ]
-            ),
-        )
-        return fig

supervisely/nn/benchmark/visualization/vis_metrics/explorer_grid.py

@@ -1,144 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Optional
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-from supervisely.project.project_meta import ProjectMeta
-
-
-class ExplorerGrid(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.clickable = True
-        self.has_diffs_view = True
-
-        filters = [{"confidence": [self.f1_optimal_conf, 1]}]
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_explorer=Widget.Markdown(title="Explore Predictions", is_header=True),
-            gallery=Widget.Gallery(filters=filters),
-        )
-
-    def _get_gallery(self, widget: Widget.Gallery, limit: Optional[int] = None) -> dict:
-        res = {}
-        api = self._loader._api
-        pred_project_id = self._loader.dt_project_info.id
-        pred_dataset = api.dataset.get_list(pred_project_id)[0]
-        project_meta = ProjectMeta.from_json(api.project.get_meta(pred_project_id))
-        pred_image_infos = api.image.get_list(dataset_id=pred_dataset.id, limit=limit)
-        pred_image_ids = [x.id for x in pred_image_infos]
-        ann_infos = api.annotation.download_batch(pred_dataset.id, pred_image_ids)
-
-        for idx, (pred_image, ann_info) in enumerate(zip(pred_image_infos, ann_infos)):
-            image_name = pred_image.name
-            image_url = pred_image.preview_url
-            widget.gallery.append(
-                title=image_name,
-                image_url=image_url,
-                annotation_info=ann_info,
-                column_index=idx % 3,
-                project_meta=project_meta,
-                ignore_tags_filtering=["outcome"],
-            )
-        res.update(widget.gallery.get_json_state())
-        res.update(widget.gallery.get_json_data()["content"])
-        res["layoutData"] = res.pop("annotations")
-        res["projectMeta"] = project_meta.to_json()
-
-        return res
-
-    def get_gallery(self, widget: Widget.Gallery):
-        return self._get_gallery(widget, limit=9)
-
-    def get_gallery_click_data(self, widget: Widget.Gallery):
-        res = {}
-
-        res["layoutTemplate"] = [{"skipObjectTagsFiltering": ["outcome"]}] * 3
-        click_data = res.setdefault("clickData", {})
-        explore = click_data.setdefault("explore", {})
-        explore["title"] = "Explore all predictions"
-        images_ids = explore.setdefault("imagesIds", [])
-
-        images_ids.extend([cd.pred_image_info.id for cd in self._loader.comparison_data.values()])
-
-        return res
-
-    def get_diff_gallery_data(self, widget: Widget.Gallery) -> Optional[dict]:
-        res = {}
-
-        res["layoutTemplate"] = [
-            {"skipObjectTagsFiltering": True, "columnTitle": "Ground Truth"},
-            {"skipObjectTagsFiltering": ["outcome"], "columnTitle": "Prediction"},
-            {"skipObjectTagsFiltering": ["confidence"], "columnTitle": "Difference"},
-        ]
-
-        click_data = res.setdefault("clickData", {})
-
-        default_filters = [
-            {"type": "tag", "tagId": "confidence", "value": [self.f1_optimal_conf, 1]},
-            # {"type": "tag", "tagId": "outcome", "value": "FP"},
-        ]
-        for img_comparison_data in self._loader.comparison_data.values():
-            gt = img_comparison_data.gt_image_info
-            pred = img_comparison_data.pred_image_info
-            diff = img_comparison_data.diff_image_info
-            assert gt.name == pred.name == diff.name
-            key = click_data.setdefault(str(pred.id), {})
-            key["imagesIds"] = [gt.id, pred.id, diff.id]
-            key["filters"] = default_filters
-            key["title"] = f"Image: {pred.name}"
-            image_id = pred.id
-            ann_json = img_comparison_data.pred_annotation.to_json()
-            assert image_id == pred.id
-            object_bindings = []
-            for obj in ann_json["objects"]:
-                for tag in obj["tags"]:
-                    if tag["name"] == "matched_gt_id":
-                        object_bindings.append(
-                            [
-                                {
-                                    "id": obj["id"],
-                                    "annotationKey": image_id,
-                                },
-                                {
-                                    "id": int(tag["value"]),
-                                    "annotationKey": gt.id,
-                                },
-                            ]
-                        )
-
-            image_id = diff.id
-            ann_json = img_comparison_data.diff_annotation.to_json()
-            assert image_id == diff.id
-            for obj in ann_json["objects"]:
-                for tag in obj["tags"]:
-                    if tag["name"] == "matched_gt_id":
-                        object_bindings.append(
-                            [
-                                {
-                                    "id": obj["id"],
-                                    "annotationKey": image_id,
-                                },
-                                {
-                                    "id": int(tag["value"]),
-                                    "annotationKey": pred.id,
-                                },
-                            ]
-                        )
-            key["objectsBindings"] = object_bindings
-
-        return res
-
-    # def get_gallery_modal(self, widget: Widget.Gallery):
-    #     res = self.get_gallery(widget)
-
-    #     res.pop("layout")
-    #     res.pop("layoutData")
-
-    #     return res

supervisely/nn/benchmark/visualization/vis_metrics/frequently_confused.py

@@ -1,115 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Optional
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class FrequentlyConfused(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-
-        self.clickable: bool = True
-        self.switchable: bool = True
-        self._keypair_sep: str = " - "
-        df = self._loader.mp.frequently_confused()
-        if df.empty:
-            self.schema = Schema(
-                self._loader.vis_texts,
-                empty=Widget.Markdown(
-                    title="Frequently Confused Classes",
-                    is_header=True,
-                    formats=[
-                        "Frequently Confused Classes",
-                        "No frequently confused class pairs found",
-                    ],
-                ),
-            )
-            self.empty = True
-            return
-
-        pair = df["category_pair"][0]
-        prob = df["probability"][0]
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_frequently_confused=Widget.Markdown(
-                title="Frequently Confused Classes",
-                is_header=True,
-                formats=[
-                    pair[0],
-                    pair[1],
-                    prob.round(2),
-                    pair[0],
-                    pair[1],
-                    (prob * 100).round(),
-                    pair[0],
-                    pair[1],
-                    pair[1],
-                    pair[0],
-                ],
-            ),
-            chart_01=Widget.Chart(switch_key="probability"),
-            chart_02=Widget.Chart(switch_key="count"),
-        )
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[Tuple[go.Figure]]:
-        if self.empty:
-            return
-
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        # Frequency of confusion as bar chart
-        confused_df = self._loader.mp.frequently_confused()
-        confused_name_pairs = confused_df["category_pair"]
-        x_labels = [f"{pair[0]} - {pair[1]}" for pair in confused_name_pairs]
-        y_labels = confused_df[widget.switch_key]
-
-        fig = go.Figure()
-        fig.add_trace(
-            go.Bar(x=x_labels, y=y_labels, marker=dict(color=y_labels, colorscale="Reds"))
-        )
-        fig.update_layout(
-            # title="Frequently confused class pairs",
-            xaxis_title="Class Pair",
-            yaxis_title=y_labels.name.capitalize(),
-        )
-        fig.update_traces(text=y_labels.round(2))
-        fig.update_traces(
-            hovertemplate="Class Pair: %{x}<br>"
-            + y_labels.name.capitalize()
-            + ": %{y:.2f}<extra></extra>"
-        )
-        return fig
-
-    def get_click_data(self, widget: Widget.Chart) -> Optional[dict]:
-        if not self.clickable or self.empty:
-            return
-        res = dict(projectMeta=self._loader.dt_project_meta.to_json())
-
-        res["layoutTemplate"] = [None, None, None]
-        res["clickData"] = {}
-
-        for keypair, v in self._loader.click_data.frequently_confused.items():
-            subkey1, subkey2 = keypair
-            key = subkey1 + self._keypair_sep + subkey2
-            res["clickData"][key] = {}
-            res["clickData"][key]["imagesIds"] = []
-            res["clickData"][key]["title"] = f"Confused classes: {subkey1} - {subkey2}"
-
-            img_ids = set()
-            obj_ids = set()
-            for x in v:
-                img_ids.add(x["dt_img_id"])
-                obj_ids.add(x["dt_obj_id"])
-
-            res["clickData"][key]["imagesIds"] = list(img_ids)
-            res["clickData"][key]["filters"] = [
-                {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
-            ]
-
-        return res

supervisely/nn/benchmark/visualization/vis_metrics/iou_distribution.py

@@ -1,86 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from supervisely.nn.benchmark.cv_tasks import CVTask
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class IOUDistribution(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        title = "Localization Accuracy (IoU)"
-        if self._loader.cv_task in [CVTask.INSTANCE_SEGMENTATION, CVTask.SEMANTIC_SEGMENTATION]:
-            title = "Mask Accuracy (IoU)"
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_localization_accuracy=Widget.Markdown(
-                title=title,
-                is_header=True,
-                formats=[self._loader.vis_texts.definitions.iou_score],
-            ),
-            markdown_iou_distribution=Widget.Markdown(
-                title="IoU Distribution",
-                is_header=True,
-                formats=[self._loader.vis_texts.definitions.iou_score],
-            ),
-            notification_avg_iou=Widget.Notification(
-                formats_title=[self._loader.base_metrics()["iou"].round(2)]
-            ),
-            chart=Widget.Chart(),
-            collapse_iou=Widget.Collapse(
-                Schema(
-                    self._loader.vis_texts,
-                    markdown_iou_calculation=Widget.Markdown(title="How IoU is calculated?"),
-                )
-            ),
-        )
-
-    def get_figure(self, widget: Widget):  # -> Optional[go.Figure]:
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        fig = go.Figure()
-        nbins = 40
-        fig.add_trace(go.Histogram(x=self._loader.mp.ious, nbinsx=nbins))
-        fig.update_layout(
-            # title="IoU Distribution",
-            xaxis_title="IoU",
-            yaxis_title="Count",
-            width=600,
-            height=500,
-        )
-
-        # Add annotation for mean IoU as vertical line
-        mean_iou = self._loader.mp.ious.mean()
-        y1 = len(self._loader.mp.ious) // nbins
-        fig.add_shape(
-            type="line",
-            x0=mean_iou,
-            x1=mean_iou,
-            y0=0,
-            y1=y1,
-            line=dict(color="orange", width=2, dash="dash"),
-        )
-        fig.update_traces(hovertemplate="IoU: %{x:.2f}<br>Count: %{y}<extra></extra>")
-        fig.add_annotation(x=mean_iou, y=y1, text=f"Mean IoU: {mean_iou:.2f}", showarrow=False)
-        fig.update_layout(
-            dragmode=False,
-            modebar=dict(
-                remove=[
-                    "zoom2d",
-                    "pan2d",
-                    "select2d",
-                    "lasso2d",
-                    "zoomIn2d",
-                    "zoomOut2d",
-                    "autoScale2d",
-                    "resetScale2d",
-                ]
-            ),
-        )
-        return fig

supervisely/nn/benchmark/visualization/vis_metrics/outcome_counts.py

@@ -1,119 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Optional
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class OutcomeCounts(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-
-        self.clickable: bool = True
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_outcome_counts=Widget.Markdown(
-                title="Outcome Counts",
-                is_header=True,
-                formats=[
-                    self._loader.vis_texts.definitions.true_positives,
-                    self._loader.vis_texts.definitions.false_positives,
-                    self._loader.vis_texts.definitions.false_negatives,
-                ],
-            ),
-            chart=Widget.Chart(),
-        )
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[go.Figure]:
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        fig = go.Figure()
-        fig.add_trace(
-            go.Bar(
-                x=[self._loader.mp.TP_count],
-                y=["Outcome"],
-                name="TP",
-                orientation="h",
-                marker=dict(color="#8ACAA1"),
-                hovertemplate="TP: %{x} objects<extra></extra>",
-            )
-        )
-        fig.add_trace(
-            go.Bar(
-                x=[self._loader.mp.FN_count],
-                y=["Outcome"],
-                name="FN",
-                orientation="h",
-                marker=dict(color="#dd3f3f"),
-                hovertemplate="FN: %{x} objects<extra></extra>",
-            )
-        )
-        fig.add_trace(
-            go.Bar(
-                x=[self._loader.mp.FP_count],
-                y=["Outcome"],
-                name="FP",
-                orientation="h",
-                marker=dict(color="#F7ADAA"),
-                hovertemplate="FP: %{x} objects<extra></extra>",
-            )
-        )
-        fig.update_layout(
-            barmode="stack",
-            width=600,
-            height=300,
-        )
-        fig.update_xaxes(title_text="Count (objects)")
-        fig.update_yaxes(tickangle=-90)
-
-        fig.update_layout(
-            dragmode=False,
-            modebar=dict(
-                remove=[
-                    "zoom2d",
-                    "pan2d",
-                    "select2d",
-                    "lasso2d",
-                    "zoomIn2d",
-                    "zoomOut2d",
-                    "autoScale2d",
-                    "resetScale2d",
-                ]
-            ),
-        )
-        return fig
-
-    def get_click_data(self, widget: Widget.Chart) -> Optional[dict]:
-        if not self.clickable:
-            return
-        res = {}
-
-        res["layoutTemplate"] = [None, None, None]
-        res["clickData"] = {}
-        for outcome, matches_data in self._loader.click_data.outcome_counts.items():
-            res["clickData"][outcome] = {}
-            res["clickData"][outcome]["imagesIds"] = []
-
-            img_ids = set()
-            for match_data in matches_data:
-                img_comparison_data = self._loader.comparison_data[match_data["gt_img_id"]]
-                if outcome == "FN":
-                    img_ids.add(img_comparison_data.diff_image_info.id)
-                else:
-                    img_ids.add(img_comparison_data.pred_image_info.id)
-
-            res["clickData"][outcome][
-                "title"
-            ] = f"{outcome}: {len(matches_data)} object{'s' if len(matches_data) > 1 else ''}"
-            res["clickData"][outcome]["imagesIds"] = list(img_ids)
-            res["clickData"][outcome]["filters"] = [
-                {"type": "tag", "tagId": "confidence", "value": [0, 1]},
-                {"type": "tag", "tagId": "outcome", "value": outcome},
-            ]
-
-        return res