supervisely 6.73.213__py3-none-any.whl → 6.73.215__py3-none-any.whl

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/calibration_score.py

@@ -0,0 +1,224 @@
+from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
+    BaseVisMetric,
+)
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    CollapseWidget,
+    MarkdownWidget,
+    NotificationWidget,
+    TableWidget,
+)
+
+
+class CalibrationScore(BaseVisMetric):
+    @property
+    def header_md(self) -> MarkdownWidget:
+        text_template = self.vis_texts.markdown_calibration_score_1
+        text = text_template.format(self.vis_texts.definitions.confidence_score)
+        return MarkdownWidget(
+            name="markdown_calibration_score",
+            title="Calibration Score",
+            text=text,
+        )
+
+    @property
+    def collapse_tip(self) -> CollapseWidget:
+        md = MarkdownWidget(
+            name="what_is_calibration",
+            title="What is calibration?",
+            text=self.vis_texts.markdown_what_is_calibration,
+        )
+        return CollapseWidget([md])
+
+    @property
+    def header_md_2(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            name="markdown_calibration_score_2",
+            title="",
+            text=self.vis_texts.markdown_calibration_score_2,
+        )
+
+    @property
+    def table(self) -> TableWidget:
+        columns = [" ", "confidence threshold", "ECE", "MCE"]
+        columns_options = [
+            {"disableSort": True},
+            {"disableSort": True},
+            {"disableSort": True},
+            {"disableSort": True},
+        ]
+        content = []
+        for i, eval_result in enumerate(self.eval_results):
+            name = f"[{i+1}] {eval_result.name}"
+            conf_threshold = eval_result.mp.m_full.get_f1_optimal_conf()[0] or 0.0
+            ece = eval_result.mp.m_full.expected_calibration_error()
+            mce = eval_result.mp.m_full.maximum_calibration_error()
+            row = [name, round(conf_threshold, 2), round(ece, 2), round(mce, 2)]
+            dct = {
+                "row": row,
+                "id": name,
+                "items": row,
+            }
+            content.append(dct)
+        data = {
+            "columns": columns,
+            "columnsOptions": columns_options,
+            "content": content,
+        }
+        return TableWidget(
+            "table_reliability",
+            data,
+            show_header_controls=False,
+            # main_column=columns[0],
+            fix_columns=1,
+        )
+
+    @property
+    def reliability_diagram_md(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            name="markdown_reliability_diagram",
+            title="Reliability Diagram",
+            text=self.vis_texts.markdown_reliability_diagram,
+        )
+
+    @property
+    def reliability_chart(self) -> ChartWidget:
+        return ChartWidget(name="chart_reliability", figure=self.get_rel_figure())
+
+    @property
+    def collapse_ece(self) -> CollapseWidget:
+        md = MarkdownWidget(
+            name="markdown_calibration_curve_interpretation",
+            title="How to interpret the Calibration curve",
+            text=self.vis_texts.markdown_calibration_curve_interpretation,
+        )
+        return CollapseWidget([md])
+
+    @property
+    def confidence_score_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_confidence_score_1.format(
+            self.vis_texts.definitions.confidence_threshold
+        )
+        return MarkdownWidget(
+            "markdown_confidence_score_1",
+            "Confidence Score Profile",
+            text,
+        )
+
+    @property
+    def confidence_chart(self) -> ChartWidget:
+        return ChartWidget(name="chart_confidence", figure=self.get_conf_figure())
+
+    @property
+    def confidence_score_md_2(self) -> MarkdownWidget:
+        return MarkdownWidget(
+            name="markdown_confidence_score_2",
+            title="",
+            text=self.vis_texts.markdown_confidence_score_2,
+        )
+
+    @property
+    def collapse_conf_score(self) -> CollapseWidget:
+        md = MarkdownWidget(
+            name="markdown_plot_confidence_profile",
+            title="How to plot Confidence Profile?",
+            text=self.vis_texts.markdown_plot_confidence_profile,
+        )
+        return CollapseWidget([md])
+
+    def get_conf_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        # Create an empty figure
+        fig = go.Figure()
+
+        for i, eval_result in enumerate(self.eval_results):
+            # Add a line trace for each eval_result
+            fig.add_trace(
+                go.Scatter(
+                    x=eval_result.dfsp_down["scores"],
+                    y=eval_result.dfsp_down["f1"],
+                    mode="lines",
+                    name=f"[{i+1}] {eval_result.name}",
+                    line=dict(color=eval_result.color),
+                    hovertemplate="Confidence Score: %{x:.2f}<br>Value: %{y:.2f}<extra></extra>",
+                )
+            )
+
+            # Add a vertical line and annotation for F1-optimal threshold if available
+            if eval_result.mp.f1_optimal_conf is not None and eval_result.mp.best_f1 is not None:
+                fig.add_shape(
+                    type="line",
+                    x0=eval_result.mp.f1_optimal_conf,
+                    x1=eval_result.mp.f1_optimal_conf,
+                    y0=0,
+                    y1=eval_result.mp.best_f1,
+                    line=dict(color="gray", width=2, dash="dash"),
+                    name=f"F1-optimal threshold ({eval_result.name})",
+                )
+
+        # Update the layout
+        fig.update_layout(
+            yaxis=dict(range=[0, 1], title="Scores"),
+            xaxis=dict(range=[0, 1], tick0=0, dtick=0.1, title="Confidence Score"),
+            height=500,
+            dragmode=False,
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+            showlegend=True,  # Show legend to differentiate between results
+        )
+
+        return fig
+
+    def get_rel_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+
+        fig = go.Figure()
+
+        for i, eval_result in enumerate(self.eval_results):
+            # Calibration curve (only positive predictions)
+            true_probs, pred_probs = eval_result.mp.m_full.calibration_curve()
+
+            fig.add_trace(
+                go.Scatter(
+                    x=pred_probs,
+                    y=true_probs,
+                    mode="lines+markers",
+                    name=f"[{i+1}] {eval_result.name}",
+                    line=dict(color=eval_result.color),
+                    hovertemplate=f"{eval_result.name}<br>"
+                    + "Confidence Score: %{x:.2f}<br>Fraction of True Positives: %{y:.2f}<extra></extra>",
+                )
+            )
+
+        fig.add_trace(
+            go.Scatter(
+                x=[0, 1],
+                y=[0, 1],
+                mode="lines",
+                name="Perfectly calibrated",
+                line=dict(color="orange", dash="dash"),
+            )
+        )
+
+        fig.update_layout(
+            # title="Calibration Curve (only positive predictions)",
+            xaxis_title="Confidence Score",
+            yaxis_title="Fraction of True Positives",
+            legend=dict(x=0.6, y=0.1),
+            xaxis=dict(range=[0, 1.1]),
+            yaxis=dict(range=[0, 1.1]),
+            width=700,
+            height=500,
+        )
+        return fig

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/explore_predicttions.py

@@ -0,0 +1,112 @@
+from typing import List, Tuple
+
+from supervisely.annotation.annotation import Annotation
+from supervisely.api.image_api import ImageInfo
+from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
+    BaseVisMetric,
+)
+from supervisely.nn.benchmark.visualization.widgets import GalleryWidget, MarkdownWidget
+from supervisely.project.project_meta import ProjectMeta
+
+
+class ExplorePredictions(BaseVisMetric):
+
+    MARKDOWN_DIFFERENCE = "markdown_explore_difference"
+    GALLERY_DIFFERENCE = "explore_difference_gallery"
+    MARKDOWN_SAME_ERRORS = "markdown_explore_same_errors"
+    GALLERY_SAME_ERRORS = "explore_same_error_gallery"
+
+    @property
+    def difference_predictions_md(self) -> MarkdownWidget:
+        text = self.vis_texts.markdown_explore_difference
+        return MarkdownWidget(self.MARKDOWN_DIFFERENCE, "Explore Predictions", text)
+
+    @property
+    def explore_gallery(self) -> GalleryWidget:
+        columns_number = len(self.eval_results) + 1
+        *data, min_conf = self._get_sample_data()
+        default_filters = [{"confidence": [min_conf, 1]}]
+        gallery = GalleryWidget(
+            self.GALLERY_DIFFERENCE, columns_number=columns_number, filters=default_filters
+        )
+        gallery.add_image_left_header("Click to explore more")
+        gallery.show_all_button = True
+        gallery.set_project_meta(self.eval_results[0].gt_project_meta)
+        gallery.set_images(*data)
+        gallery.add_on_click(
+            self.explore_modal_table.id, self.get_click_data_explore_all(), columns_number * 3
+        )
+        gallery._gallery._filters
+        gallery._gallery._update_filters()
+
+        return gallery
+
+    def _get_sample_data(self) -> Tuple[List[ImageInfo], List[Annotation], List[ProjectMeta]]:
+        images = []
+        annotations = []
+        metas = [self.eval_results[0].gt_project_meta]
+        skip_tags_filtering = []
+        api = self.eval_results[0].api
+        min_conf = float("inf")
+        for idx, eval_res in enumerate(self.eval_results):
+            if idx == 0:
+                dataset_info = api.dataset.get_list(eval_res.gt_project_id)[0]
+                image_infos = api.image.get_list(dataset_info.id, limit=5)
+                images_ids = [image_info.id for image_info in image_infos]
+                images.append(image_infos)
+                anns = api.annotation.download_batch(dataset_info.id, images_ids)
+                annotations.append(anns)
+                skip_tags_filtering.append(True)
+            metas.append(eval_res.dt_project_meta)
+            dataset_info = api.dataset.get_list(eval_res.dt_project_id)[0]
+            image_infos = eval_res.api.image.get_list(dataset_info.id, limit=5)
+            images_ids = [image_info.id for image_info in image_infos]
+            images.append(image_infos)
+            anns = eval_res.api.annotation.download_batch(dataset_info.id, images_ids)
+            annotations.append(anns)
+            skip_tags_filtering.append(False)
+            min_conf = min(min_conf, eval_res.f1_optimal_conf)
+
+        images = list(i for x in zip(*images) for i in x)
+        annotations = list(i for x in zip(*annotations) for i in x)
+        return images, annotations, metas, skip_tags_filtering, min_conf
+
+    def get_click_data_explore_all(self) -> dict:
+        res = {}
+
+        res["projectMeta"] = self.eval_results[0].gt_project_meta.to_json()
+        res["layoutTemplate"] = [None, None, None]
+
+        res["layoutTemplate"] = [{"skipObjectTagsFiltering": True, "columnTitle": "Ground Truth"}]
+        for i in range(len(self.eval_results)):
+            res["layoutTemplate"].append({"columnTitle": f"Model {i + 1}"})
+
+        click_data = res.setdefault("clickData", {})
+        explore = click_data.setdefault("explore", {})
+        explore["title"] = "Explore all predictions"
+
+        images_ids = []
+        api = self.eval_results[0].api
+        min_conf = float("inf")
+        for idx, eval_res in enumerate(self.eval_results):
+            if idx == 0:
+                dataset_infos = api.dataset.get_list(eval_res.gt_project_id)
+                current_images_ids = []
+                for ds in dataset_infos:
+                    image_infos = eval_res.api.image.get_list(ds.id)
+                    current_images_ids.extend([image_info.id for image_info in image_infos])
+                images_ids.append(current_images_ids)
+
+            current_images_ids = []
+            dataset_infos = api.dataset.get_list(eval_res.dt_project_id)
+            for ds in dataset_infos:
+                image_infos = eval_res.api.image.get_list(ds.id)
+                current_images_ids.extend([image_info.id for image_info in image_infos])
+            images_ids.append(current_images_ids)
+
+            min_conf = min(min_conf, eval_res.f1_optimal_conf)
+
+        explore["imagesIds"] = list(i for x in zip(*images_ids) for i in x)
+        explore["filters"] = [{"type": "tag", "tagId": "confidence", "value": [min_conf, 1]}]
+
+        return res

supervisely/nn/benchmark/comparison/detection_visualization/vis_metrics/localization_accuracy.py

@@ -0,0 +1,161 @@
+import numpy as np
+
+from supervisely.nn.benchmark.comparison.detection_visualization.vis_metrics.vis_metric import (
+    BaseVisMetric,
+)
+from supervisely.nn.benchmark.cv_tasks import CVTask
+from supervisely.nn.benchmark.visualization.widgets import (
+    ChartWidget,
+    CollapseWidget,
+    MarkdownWidget,
+    TableWidget,
+)
+
+
+class LocalizationAccuracyIoU(BaseVisMetric):
+    @property
+    def header_md(self) -> MarkdownWidget:
+        title = "Localization Accuracy (IoU)"
+        if self.eval_results[0].cv_task in [
+            CVTask.INSTANCE_SEGMENTATION,
+            CVTask.SEMANTIC_SEGMENTATION,
+        ]:
+            title = "Mask Accuracy (IoU)"
+        text_template = self.vis_texts.markdown_localization_accuracy
+        text = text_template.format(self.vis_texts.definitions.iou_score)
+        return MarkdownWidget(
+            name="markdown_localization_accuracy",
+            title=title,
+            text=text,
+        )
+
+    @property
+    def iou_distribution_md(self) -> MarkdownWidget:
+        text_template = self.vis_texts.markdown_iou_distribution
+        text = text_template.format(self.vis_texts.definitions.iou_score)
+        return MarkdownWidget(
+            name="markdown_iou_distribution",
+            title="IoU Distribution",
+            text=text,
+        )
+
+    @property
+    def table_widget(self) -> TableWidget:
+        res = {}
+
+        columns = [" ", "Avg. IoU"]
+        res["content"] = []
+        for i, eval_result in enumerate(self.eval_results, 1):
+            value = round(eval_result.mp.base_metrics()["iou"], 2)
+            model_name = f"[{i}] {eval_result.name}"
+            row = [model_name, value]
+            dct = {
+                "row": row,
+                "id": model_name,
+                "items": row,
+            }
+            res["content"].append(dct)
+
+        columns_options = [{"disableSort": True}, {"disableSort": True}]
+
+        res["columns"] = columns
+        res["columnsOptions"] = columns_options
+
+        return TableWidget(
+            name="localization_accuracy_table", data=res, show_header_controls=False, fix_columns=1
+        )
+
+    @property
+    def chart(self) -> ChartWidget:
+        return ChartWidget(name="chart_iou_distribution", figure=self.get_figure())
+
+    @property
+    def collapse_tip(self) -> CollapseWidget:
+        inner_md = MarkdownWidget(
+            name="markdown_iou_calculation",
+            title="How IoU is calculated?",
+            text=self.vis_texts.markdown_iou_calculation,
+        )
+        return CollapseWidget(widgets=[inner_md])
+
+    def get_figure(self):
+        import plotly.graph_objects as go  # pylint: disable=import-error
+        from scipy.stats import gaussian_kde  # pylint: disable=import-error
+
+        fig = go.Figure()
+        nbins = 40
+        # min_value = min([r.mp.ious[0] for r in self.eval_results])
+        x_range = np.linspace(0.5, 1, 500)
+        hist_data = [np.histogram(r.mp.ious, bins=nbins) for r in self.eval_results]
+        bin_width = min([bin_edges[1] - bin_edges[0] for _, bin_edges in hist_data])
+
+        for i, (eval_result, (hist, bin_edges)) in enumerate(zip(self.eval_results, hist_data)):
+            name = f"[{i+1}] {eval_result.name}"
+            kde = gaussian_kde(eval_result.mp.ious)
+            density = kde(x_range)
+
+            scaling_factor = len(eval_result.mp.ious) * bin_width
+            scaled_density = density * scaling_factor
+
+            fig.add_trace(
+                go.Bar(
+                    x=bin_edges[:-1],
+                    y=hist,
+                    width=bin_width,
+                    name=f"{name} (Bars)",
+                    offset=0,
+                    opacity=0.2,
+                    hovertemplate=name + "<br>IoU: %{x:.2f}<br>Count: %{y}<extra></extra>",
+                    marker=dict(color=eval_result.color, line=dict(width=0)),
+                )
+            )
+
+            fig.add_trace(
+                go.Scatter(
+                    x=x_range,
+                    y=scaled_density,
+                    name=f"{name} (KDE)",
+                    line=dict(color=eval_result.color, width=2),
+                    hovertemplate=name + "<br>IoU: %{x:.2f}<br>Count: %{y:.1f}<extra></extra>",
+                )
+            )
+
+        fig.update_layout(
+            xaxis_title="IoU",
+            yaxis_title="Count",
+        )
+
+        # Add annotation for mean IoU as vertical line
+        for i, eval_result in enumerate(self.eval_results):
+            mean_iou = eval_result.mp.ious.mean()
+            y1 = len(eval_result.mp.ious) // nbins
+            fig.add_shape(
+                type="line",
+                x0=mean_iou,
+                x1=mean_iou,
+                y0=0,
+                y1=y1,
+                line=dict(color="orange", width=2, dash="dash"),
+            )
+
+        fig.update_layout(
+            barmode="overlay",
+            bargap=0,
+            bargroupgap=0,
+            dragmode=False,
+            yaxis=dict(rangemode="tozero"),
+            xaxis=dict(range=[0.5, 1]),
+            modebar=dict(
+                remove=[
+                    "zoom2d",
+                    "pan2d",
+                    "select2d",
+                    "lasso2d",
+                    "zoomIn2d",
+                    "zoomOut2d",
+                    "autoScale2d",
+                    "resetScale2d",
+                ]
+            ),
+        )
+        return fig