supervisely 6.73.238__py3-none-any.whl → 6.73.240__py3-none-any.whl

supervisely/nn/benchmark/visualization/vis_metrics/outcome_counts_per_class.py

@@ -1,148 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Optional
-
-import numpy as np
-import pandas as pd
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class PerClassOutcomeCounts(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.clickable: bool = True
-        self.switchable: bool = True
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_class_outcome_counts_1=Widget.Markdown(
-                title="Outcome Counts by Class",
-                is_header=True,
-                formats=[
-                    self._loader.vis_texts.definitions.true_positives,
-                    self._loader.vis_texts.definitions.false_positives,
-                    self._loader.vis_texts.definitions.false_negatives,
-                ],
-            ),
-            markdown_class_outcome_counts_2=Widget.Markdown(
-                formats=[self._loader.vis_texts.definitions.f1_score]
-            ),
-            collapse_perclass_outcome=Widget.Collapse(
-                Schema(
-                    self._loader.vis_texts,
-                    markdown_normalization=Widget.Markdown(title="Normalization"),
-                )
-            ),
-            chart_01=Widget.Chart(switch_key="normalized"),
-            chart_02=Widget.Chart(switch_key="absolute"),
-        )
-
-    def get_figure(self, widget: Widget.Chart):  #  -> Optional[go.Figure]:
-        import plotly.express as px  # pylint: disable=import-error
-
-        # Per-class Counts
-        iou_thres = 0
-
-        tp = self._loader.mp.true_positives[:, iou_thres]
-        fp = self._loader.mp.false_positives[:, iou_thres]
-        fn = self._loader.mp.false_negatives[:, iou_thres]
-
-        # normalize
-        support = tp + fn
-        with np.errstate(invalid="ignore", divide="ignore"):
-            tp_rel = tp / support
-            fp_rel = fp / support
-            fn_rel = fn / support
-
-            # sort by f1
-            sort_scores = 2 * tp / (2 * tp + fp + fn)
-
-        K = len(self._loader.mp.cat_names)
-        sort_indices = np.argsort(sort_scores)
-        cat_names_sorted = [self._loader.mp.cat_names[i] for i in sort_indices]
-        tp_rel, fn_rel, fp_rel = tp_rel[sort_indices], fn_rel[sort_indices], fp_rel[sort_indices]
-
-        objects_count = np.concatenate([tp[sort_indices], fn[sort_indices], fp[sort_indices]])
-        data = {
-            "Type": ["TP"] * K + ["FN"] * K + ["FP"] * K,
-            "category": cat_names_sorted * 3,
-        }
-        y_label = ""
-        if widget.switch_key == "normalized":
-            y_label = "Objects Fraction"
-            # Stacked per-class counts
-            data["count"] = np.concatenate([tp_rel, fn_rel, fp_rel])
-        elif widget.switch_key == "absolute":
-            y_label = "Objects Count"
-            data["count"] = objects_count
-
-        df = pd.DataFrame(data)
-
-        color_map = {"TP": "#8ACAA1", "FN": "#dd3f3f", "FP": "#F7ADAA"}
-        fig = px.bar(
-            df,
-            x="category",
-            y="count",
-            color="Type",
-            # title="Per-class Outcome Counts",
-            height=500,
-            width=1000,
-            labels={"count": y_label, "category": "Class"},
-            color_discrete_map=color_map,
-        )
-        xaxis_title = fig.layout.xaxis.title.text
-        yaxis_title = fig.layout.yaxis.title.text
-        if widget.switch_key == "normalized":
-
-            fig.update_traces(
-                hovertemplate="Type=%{fullData.name} <br>"
-                + xaxis_title
-                + "=%{x}<br>"
-                + yaxis_title
-                + "=%{y:.2f}<extra></extra>"
-                # "Images count=%{y:.2f}<extra></extra>"
-            )
-        elif widget.switch_key == "absolute":
-            fig.update_traces(
-                hovertemplate="Type=%{fullData.name} <br>"
-                + xaxis_title
-                + "=%{x}<br>"
-                + yaxis_title
-                + "=%{y}<extra></extra>",
-            )
-        return fig
-
-    def get_click_data(self, widget: Widget.Chart) -> Optional[dict]:
-        if not self.clickable:
-            return
-        res = {}
-        res["layoutTemplate"] = [None, None, None]
-
-        res["clickData"] = {}
-        for class_name, v1 in self._loader.click_data.outcome_counts_by_class.items():
-            for outcome, matches_data in v1.items():
-                key = class_name + self._keypair_sep + outcome
-                res["clickData"][key] = {}
-                res["clickData"][key]["imagesIds"] = []
-                res["clickData"][key][
-                    "title"
-                ] = f"Images with objects of class '{class_name}' and outcome '{outcome}'"
-
-                img_ids = set()
-                for match_data in matches_data:
-                    img_comparison_data = self._loader.comparison_data[match_data["gt_img_id"]]
-                    if outcome == "FN":
-                        img_ids.add(img_comparison_data.diff_image_info.id)
-                    else:
-                        img_ids.add(img_comparison_data.pred_image_info.id)
-                res["clickData"][key]["imagesIds"] = list(img_ids)
-                res["clickData"][key]["filters"] = [
-                    {"type": "tag", "tagId": "confidence", "value": [0, 1]},
-                    {"type": "tag", "tagId": "outcome", "value": outcome},
-                ]
-        return res

supervisely/nn/benchmark/visualization/vis_metrics/overall_error_analysis.py

@@ -1,109 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List
-
-from supervisely.nn.benchmark.cv_tasks import CVTask
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class OverallErrorAnalysis(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.cv_tasks: List[CVTask] = [CVTask.SEMANTIC_SEGMENTATION.value]
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[go.Figure]:
-        import plotly.graph_objects as go  # pylint: disable=import-error
-        from plotly.subplots import make_subplots  # pylint: disable=import-error
-
-        fig = make_subplots(
-            rows=1,
-            cols=3,
-            subplot_titles=(
-                "Basic segmentation metrics",
-                "Intersection & Error over Union",
-                "Renormalized Error over Union",
-            ),
-            specs=[[{"type": "polar"}, {"type": "domain"}, {"type": "xy"}]],
-        )
-
-        # first subplot
-        categories = [
-            "mPixel accuracy",
-            "mPrecision",
-            "mRecall",
-            "mF1-score",
-            "mIoU",
-            "mBoundaryIoU",
-            "mPixel accuracy",
-        ]
-        values = [64, 60.4, 52, 51.4, 37.9, 20.5, 64]
-        trace_1 = go.Scatterpolar(
-            mode="lines+text",
-            r=values,
-            theta=categories,
-            fill="toself",
-            fillcolor="cornflowerblue",
-            line_color="blue",
-            opacity=0.6,
-            text=[64, 60.4, 52, 51.4, 37.9, 20.5, 64],
-            textposition=[
-                "bottom right",
-                "top center",
-                "top center",
-                "middle left",
-                "bottom center",
-                "bottom right",
-                "bottom right",
-            ],
-            textfont=dict(color="blue"),
-        )
-        fig.add_trace(trace_1, row=1, col=1)
-
-        # second subplot
-        labels = ["mIoU", "mBoundaryEoU", "mExtentEoU", "mSegmentEoU"]
-        values = [37.9, 13.1, 25.8, 23.2]
-        trace_2 = go.Pie(
-            labels=labels,
-            values=values,
-            hole=0.5,
-            textposition="outside",
-            textinfo="percent+label",
-            marker=dict(colors=["cornflowerblue", "moccasin", "lightgreen", "orangered"]),
-        )
-        fig.add_trace(trace_2, row=1, col=2)
-
-        # third subplot
-        labels = ["boundary", "extent", "segment"]
-        values = [28.9, 37.6, 23.2]
-        trace_3 = go.Bar(
-            x=labels,
-            y=values,
-            orientation="v",
-            text=values,
-            width=[0.5, 0.5, 0.5],
-            textposition="outside",
-            marker_color=["moccasin", "lightgreen", "orangered"],
-        )
-        fig.add_trace(trace_3, row=1, col=3)
-
-        fig.update_layout(
-            height=400,
-            width=1200,
-            polar=dict(
-                radialaxis=dict(visible=True, showline=False, showticklabels=False, range=[0, 100])
-            ),
-            showlegend=False,
-            plot_bgcolor="rgba(0, 0, 0, 0)",
-            yaxis=dict(showticklabels=False),
-            yaxis_range=[0, int(max(values)) + 4],
-        )
-        fig.layout.annotations[0].update(y=1.2)
-        fig.layout.annotations[1].update(y=1.2)
-        fig.layout.annotations[2].update(y=1.2)
-
-        return fig

supervisely/nn/benchmark/visualization/vis_metrics/overview.py

@@ -1,189 +0,0 @@
-from __future__ import annotations
-
-import datetime
-from typing import TYPE_CHECKING
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class Overview(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self._is_overview = True
-        info = loader.inference_info or {}
-        url = info.get("checkpoint_url")
-        link_text = info.get("custom_checkpoint_path")
-        if link_text is None:
-            link_text = url
-        if link_text is not None:
-            link_text = link_text.replace("_", "\_")
-
-        # Note about validation dataset
-        classes_str, note_about_val_dataset, train_session = self.get_overview_info()
-
-        checkpoint_name = info.get("deploy_params", {}).get("checkpoint_name", "")
-        me = self._loader._api.user.get_my_info().login
-        current_date = datetime.datetime.now().strftime("%d %B %Y, %H:%M")
-        model_name = info.get("model_name") or "Custom"
-
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_header=Widget.Markdown(
-                title="Header",
-                is_header=False,
-                formats=[checkpoint_name, me, current_date],  # Title
-            ),
-            markdown_overview=Widget.Markdown(
-                title="Overview",
-                is_header=True,
-                formats=[
-                    model_name.replace("_", "\_"),
-                    checkpoint_name.replace("_", "\_"),
-                    info.get("architecture"),
-                    info.get("task_type"),
-                    info.get("runtime"),
-                    url,
-                    link_text,
-                    self._loader.gt_project_info.id,
-                    self._loader.gt_project_info.name,
-                    classes_str,
-                    note_about_val_dataset,
-                    train_session,
-                    self._loader.docs_link,
-                ],
-            ),
-            markdown_key_metrics=Widget.Markdown(
-                title="Key Metrics",
-                is_header=True,
-                formats=[
-                    self._loader.vis_texts.definitions.average_precision,
-                    self._loader.vis_texts.definitions.confidence_threshold,
-                    self._loader.vis_texts.definitions.confidence_score,
-                ],
-            ),
-            table_key_metrics=Widget.Table(),
-            chart=Widget.Chart(),
-        )
-
-    def get_figure(self, widget: Widget.Chart):  #  -> Optional[go.Figure]
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        # Overall Metrics
-        base_metrics = self._loader.mp.base_metrics()
-        r = list(base_metrics.values())
-        theta = [self._loader.mp.metric_names[k] for k in base_metrics.keys()]
-        fig = go.Figure()
-        fig.add_trace(
-            go.Scatterpolar(
-                r=r + [r[0]],
-                theta=theta + [theta[0]],
-                fill="toself",
-                name="Overall Metrics",
-                hovertemplate="%{theta}: %{r:.2f}<extra></extra>",
-            )
-        )
-        fig.update_layout(
-            polar=dict(
-                radialaxis=dict(
-                    range=[0.0, 1.0],
-                    ticks="outside",
-                ),
-                angularaxis=dict(rotation=90, direction="clockwise"),
-            ),
-            dragmode=False,
-            # title="Overall Metrics",
-            # width=700,
-            # height=500,
-            # autosize=False,
-            margin=dict(l=25, r=25, t=25, b=25),
-        )
-        fig.update_layout(
-            modebar=dict(
-                remove=[
-                    "zoom2d",
-                    "pan2d",
-                    "select2d",
-                    "lasso2d",
-                    "zoomIn2d",
-                    "zoomOut2d",
-                    "autoScale2d",
-                    "resetScale2d",
-                ]
-            )
-        )
-        return fig
-
-    def get_overview_info(self):
-        classes_cnt = len(self._loader._benchmark.classes_whitelist or [])
-        classes_str = "classes" if classes_cnt > 1 else "class"
-        classes_str = f"{classes_cnt} {classes_str}"
-
-        train_session, images_str = "", ""
-        gt_project_id = self._loader.gt_project_info.id
-        gt_images_ids = self._loader._benchmark.gt_images_ids
-        gt_dataset_ids = self._loader._benchmark.gt_dataset_ids
-        train_info = self._loader._benchmark.train_info
-        if gt_images_ids is not None:
-            val_imgs_cnt = len(gt_images_ids)
-        elif gt_dataset_ids is not None:
-            datasets = [self._loader._api.dataset.get_info_by_id(ds) for ds in gt_dataset_ids]
-            val_imgs_cnt = sum(ds.items_count for ds in datasets)
-        else:
-            val_imgs_cnt = self._loader.gt_project_info.items_count
-
-        if train_info:
-            train_task_id = train_info.get("app_session_id")
-            if train_task_id:
-                task_info = self._loader._api.task.get_info_by_id(int(train_task_id))
-                app_id = task_info["meta"]["app"]["id"]
-                train_session = f'- **Training dashboard**:  <a href="/apps/{app_id}/sessions/{train_task_id}" target="_blank">open</a>'
-
-            train_imgs_cnt = train_info.get("images_count")
-            images_str = f", {train_imgs_cnt} images in train, {val_imgs_cnt} images in validation"
-
-        if gt_images_ids is not None:
-            images_str += f". Evaluated using subset - {val_imgs_cnt} images"
-        elif gt_dataset_ids is not None:
-            links = [
-                f'<a href="/projects/{gt_project_id}/datasets/{ds.id}" target="_blank">{ds.name}</a>'
-                for ds in datasets
-            ]
-            images_str += (
-                f". Evaluated on the dataset{'s' if len(links) > 1 else ''}: {', '.join(links)}"
-            )
-        else:
-            images_str += f". Evaluated on the whole project ({val_imgs_cnt} images)"
-
-        return classes_str, images_str, train_session
-
-    def get_table(self, widget: Widget.Table) -> dict:
-        res = {}
-
-        columns = ["metrics", "values"]
-        res["content"] = []
-        for metric, value in self._loader.mp.metric_table().items():
-            row = [metric, round(value, 2)]
-            dct = {
-                "row": row,
-                "id": metric,
-                "items": row,
-            }
-            res["content"].append(dct)
-
-        columns_options = [
-            {"customCell": True, "disableSort": True},
-            {"disableSort": True},
-        ]
-
-        res["columns"] = columns
-        res["columnsOptions"] = columns_options
-
-        widget.main_column = columns[0]
-        widget.show_header_controls = False
-        widget.width = "60%"
-        return res

supervisely/nn/benchmark/visualization/vis_metrics/percision_avg_per_class.py

@@ -1,57 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import numpy as np
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class PerClassAvgPrecision(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.clickable = True
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_class_ap=Widget.Markdown(
-                title="Average Precision by Class",
-                is_header=True,
-                formats=[self._loader.vis_texts.definitions.average_precision],
-            ),
-            chart=Widget.Chart(),
-        )
-
-    def get_figure(self, widget: Widget):  # -> Optional[go.Figure]:
-        import plotly.express as px  # pylint: disable=import-error
-
-        # AP per-class
-        ap_per_class = self._loader.mp.coco_precision[:, :, :, 0, 2].mean(axis=(0, 1))
-        ap_per_class[ap_per_class == -1] = 0  # -1 is a placeholder for no GT
-        labels = dict(r="Average Precision", theta="Class")
-        fig = px.scatter_polar(
-            r=ap_per_class,
-            theta=self._loader.mp.cat_names,
-            # title="Per-class Average Precision (AP)",
-            labels=labels,
-            width=800,
-            height=800,
-            range_r=[0, 1],
-        )
-        fig.update_traces(fill="toself")
-        fig.update_layout(
-            modebar_add=["resetScale"],
-            margin=dict(l=80, r=80, t=0, b=0),
-        )
-        fig.update_traces(
-            hovertemplate=labels["theta"]
-            + ": %{theta}<br>"
-            + labels["r"]
-            + ": %{r:.2f}<br>"
-            + "<extra></extra>"
-        )
-        return fig

supervisely/nn/benchmark/visualization/vis_metrics/pr_curve.py

@@ -1,101 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import numpy as np
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class PRCurve(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_pr_curve=Widget.Markdown(
-                title="Precision-Recall Curve",
-                is_header=True,
-                formats=[self._loader.vis_texts.definitions.f1_score],
-            ),
-            collapse_pr=Widget.Collapse(
-                schema=Schema(
-                    self._loader.vis_texts,
-                    markdown_trade_offs=Widget.Markdown(
-                        title="About Trade-offs between precision and recall"
-                    ),
-                    markdown_what_is_pr_curve=Widget.Markdown(
-                        title="How the PR curve is built?",
-                        formats=[
-                            self._loader.vis_texts.definitions.confidence_score,
-                            self._loader.vis_texts.definitions.true_positives,
-                            self._loader.vis_texts.definitions.false_positives,
-                        ],
-                    ),
-                )
-            ),
-            notification_ap=Widget.Notification(
-                formats_title=[loader.base_metrics()["mAP"].round(2)]
-            ),
-            chart=Widget.Chart(),
-        )
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[go.Figure]:
-        import plotly.express as px  # pylint: disable=import-error
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        # Precision-Recall curve
-        pr_curve = self._loader.mp.pr_curve().copy()
-        pr_curve[pr_curve == -1] = np.nan  # -1 is a placeholder for no GT
-        pr_curve = np.nanmean(pr_curve, axis=-1)
-        fig = px.line(
-            x=self._loader.mp.recThrs,
-            y=pr_curve,
-            # title="Precision-Recall Curve",
-            labels={"x": "Recall", "y": "Precision"},
-            width=600,
-            height=500,
-        )
-        fig.data[0].name = "Model"
-        fig.data[0].showlegend = True
-        fig.update_traces(fill="tozeroy", line=dict(color="#1f77b4"))
-        fig.add_trace(
-            go.Scatter(
-                x=self._loader.mp.recThrs,
-                y=[1] * len(self._loader.mp.recThrs),
-                name="Perfect",
-                line=dict(color="orange", dash="dash"),
-                showlegend=True,
-            )
-        )
-        fig.add_annotation(
-            text=f"mAP = {self._loader.mp.base_metrics()['mAP']:.2f}",
-            xref="paper",
-            yref="paper",
-            x=0.98,
-            y=0.92,
-            showarrow=False,
-            bgcolor="white",
-        )
-        fig.update_traces(hovertemplate="Recall: %{x:.2f}<br>Precision: %{y:.2f}<extra></extra>")
-        fig.update_layout(
-            dragmode=False,
-            modebar=dict(
-                remove=[
-                    "zoom2d",
-                    "pan2d",
-                    "select2d",
-                    "lasso2d",
-                    "zoomIn2d",
-                    "zoomOut2d",
-                    "autoScale2d",
-                    "resetScale2d",
-                ]
-            ),
-        )
-        # fig.show()
-        return fig

supervisely/nn/benchmark/visualization/vis_metrics/pr_curve_by_class.py

@@ -1,46 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pandas as pd
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class PRCurveByClass(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.clickable = True
-        self.schema = Schema(
-            self._loader.vis_texts,
-            markdown_pr_by_class=Widget.Markdown(title="PR Curve by Class"),
-            chart=Widget.Chart(),
-        )
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[go.Figure]:
-        import plotly.express as px  # pylint: disable=import-error
-
-        # Precision-Recall curve per-class
-        df = pd.DataFrame(self._loader.mp.pr_curve(), columns=self._loader.mp.cat_names)
-
-        fig = px.line(
-            df,
-            x=self._loader.mp.recThrs,
-            y=df.columns,
-            # title="Precision-Recall Curve per Class",
-            labels={"x": "Recall", "value": "Precision", "variable": "Category"},
-            color_discrete_sequence=px.colors.qualitative.Prism,
-            width=800,
-            height=600,
-        )
-
-        fig.update_yaxes(range=[0, 1])
-        fig.update_xaxes(range=[0, 1])
-        # fig.show()
-
-        return fig