supervisely 6.73.237__py3-none-any.whl → 6.73.239__py3-none-any.whl

supervisely/nn/benchmark/visualization/inference_speed/speedtest_intro.py

@@ -1,28 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class SpeedtestIntro(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        speedtest = self._loader.speedtest["speedtest"][0]
-        self.schema = Schema(
-            self._loader.inference_speed_text,
-            markdown_speedtest_intro=Widget.Markdown(
-                title="Inference Speed",
-                is_header=True,
-                formats=[
-                    speedtest["device"],
-                    self._loader.hardware,
-                    speedtest["runtime"]
-                ],
-            ),
-        )

supervisely/nn/benchmark/visualization/inference_speed/speedtest_overview.py

@@ -1,141 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class SpeedtestOverview(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.clickable = False
-        num_iterations = self._loader.speedtest["speedtest"][0]["num_iterations"]
-        self.schema = Schema(
-            self._loader.inference_speed_text,
-            markdown_speedtest_overview=Widget.Markdown(title="Info", formats=[num_iterations]),
-            table=Widget.Table(),
-        )
-        self._row_ids = None
-
-    def get_table(self, widget: Widget.Table) -> dict:
-        res = {}
-
-        columns = [" ", "Infrence time", "FPS"]
-        temp_res = {}
-        max_fps = 0
-        for test in self._loader.speedtest["speedtest"]:
-            batch_size = test["batch_size"]
-
-            ms = round(test["benchmark"]["total"], 2)
-            fps = round(1000 / test["benchmark"]["total"] * batch_size)
-            row = [batch_size, ms, fps]
-            temp_res[batch_size] = row
-            max_fps = max(max_fps, fps)
-
-        res["content"] = []
-        # sort by batch size
-        temp_res = dict(sorted(temp_res.items()))
-        for row in temp_res.values():
-            dct = {
-                "row": row,
-                "id": row[0],
-                "items": row,
-            }
-            res["content"].append(dct)
-
-        columns_options = [
-            {"customCell": True, "disableSort": True},
-            {"subtitle": "ms", "tooltip": "Milliseconds for batch images", "postfix": "ms"},
-            {
-                "subtitle": "imgs/sec",
-                "tooltip": "Frames (images) per second",
-                "postfix": "fps",
-                "maxValue": max_fps,
-            },
-        ]
-
-        res["columns"] = columns
-        res["columnsOptions"] = columns_options
-
-        widget.main_column = "Batch size"
-        widget.fixed_columns = 1
-        widget.show_header_controls = False
-        return res
-
-    def get_table_click_data(self, widget: Widget.Table) -> dict:
-        return {}
-
-
-## ========================backup (for public benchmark)==========================
-# class SpeedtestOverview(MetricVis):
-
-#     def __init__(self, loader: Visualizer) -> None:
-#         super().__init__(loader)
-#         self.clickable = False
-#         num_iterations = self._loader.speedtest["speedtest"][0]["num_iterations"]
-#         self.schema = Schema(
-#             self._loader.inference_speed_text,
-#             markdown_speedtest_overview=Widget.Markdown(
-#                 title="Overview", formats=[num_iterations, self._loader.hardware]
-#             ),
-#             table=Widget.Table(),
-#         )
-#         self._row_ids = None
-
-#     def get_table(self, widget: Widget.Table) -> dict:
-#         res = {}
-
-#         columns = [
-#             "Runtime",
-#             "Batch size 1",
-#             "Batch size 8",
-#             "Batch size 16",
-#             "Batch size 1",
-#             "Batch size 8",
-#             "Batch size 16",
-#         ]
-#         temp_res = {}
-#         for test in self._loader.speedtest["speedtest"]:
-#             device = "GPU" if "cuda" in test["device"] else "CPU"
-#             runtime = test["runtime"]
-#             batch_size = test["batch_size"]
-#             row_name = f"{device} {runtime}"
-
-#             row = temp_res.setdefault(row_name, {})
-#             row["Runtime"] = row_name
-#             row[f"Batch size {batch_size} (ms)"] = round(test["benchmark"]["total"], 2)
-#             row[f"Batch size {batch_size} (FPS)"] = round(
-#                 1000 / test["benchmark"]["total"] * batch_size
-#             )
-
-#         res["content"] = []
-#         for row in temp_res.values():
-#             dct = {
-#                 "row": row,
-#                 "id": row["Runtime"],
-#                 "items": list(row.values()),
-#             }
-#             res["content"].append(dct)
-
-#         columns_options = [
-#             {"maxWidth": "225px"},
-#             {"subtitle": "ms", "tooltip": "Milliseconds for 1 image"},
-#             {"subtitle": "ms", "tooltip": "Milliseconds for 8 images"},
-#             {"subtitle": "ms", "tooltip": "Milliseconds for 16 images"},
-#             {"subtitle": "FPS", "tooltip": "Frames (images) per second"},
-#             {"subtitle": "FPS", "tooltip": "Frames (images) per second"},
-#             {"subtitle": "FPS", "tooltip": "Frames (images) per second"},
-#         ]
-
-#         res["columns"] = columns
-#         res["columnsOptions"] = columns_options
-
-#         return res
-
-#     def get_table_click_data(self, widget: Widget.Table) -> dict:
-#         return {}

supervisely/nn/benchmark/visualization/inference_speed/speedtest_real_time.py

@@ -1,63 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from supervisely.nn.benchmark.visualization.vis_metric_base import MetricVis
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-
-class SpeedtestRealTime(MetricVis):
-
-    def __init__(self, loader: Visualizer) -> None:
-        super().__init__(loader)
-        self.switchable: bool = True
-        charts = {}
-        for batch_size in [1, 8, 16]:
-            for measure in ["ms", "fps"]:
-                key = f"{batch_size}_{measure}"
-                charts[f"chart_{key}"] = Widget.Chart(switch_key=key)
-        self.schema = Schema(
-            self._loader.inference_speed_text,
-            markdown_real_time_inference=Widget.Markdown(title="Real-time inference"),
-            **charts,
-        )
-
-    def get_figure(self, widget: Widget.Chart):  #  -> Optional[go.Figure]
-        import plotly.graph_objects as go  # pylint: disable=import-error
-
-        colors = iter(["#17becf", "#e377c2", "#bcbd22", "#ff7f0e", "#9467bd", "#2ca02c"])
-
-        batch_size, measure = widget.switch_key.split("_")
-
-        fig = go.Figure()
-        for test in self._loader.speedtest["speedtest"]:
-            device = "GPU" if "cuda" in test["device"] else "CPU"
-            runtime = test["runtime"]
-            runtime_and_device = f"{device} {runtime}"
-            bs = test["batch_size"]
-            if batch_size != bs:
-                continue
-
-            if measure == "ms":
-                total = test["benchmark"]["total"]
-            else:
-                total = round(1000 / test["benchmark"]["total"] * bs)
-
-            fig.add_trace(
-                go.Bar(
-                    y=total,
-                    x=runtime_and_device,
-                    marker=dict(color=next(colors)),
-                )
-            )
-
-
-        y_title = "Time (ms)" if measure == "ms" else "Images per second (FPS)"
-        fig.update_xaxes(title_text="Runtime")
-        fig.update_yaxes(title_text=y_title)
-        fig.update_layout(height=400, width=800)
-
-        return fig

supervisely/nn/benchmark/visualization/text_templates/inference_speed_text.py

@@ -1,23 +0,0 @@
-markdown_speedtest_intro = """## Inference Speed
-
-This is a speed test benchmark for this model. The model was tested with the following configuration:
-
-- **Device**: {}
-- **Hardware**: {}
-- **Runtime**: {}
-"""
-
-markdown_speedtest_overview = """
-The table below shows the speed test results. For each test, the time taken to process one batch of images is shown, as well as the model's throughput (i.e, the number of images processed per second, or FPS). Results are averaged across **{}** iterations.
-"""
-
-markdown_real_time_inference = """## Real-time Inference
-
-This chart compares different runtimes and devices (CPU or GPU)."""
-
-# We additionally divide **predict** procedure into three stages: pre-process, inference, and post-process. Each bar in this chart consists of these three stages. For example, in the chart you can find how long the post-process phase lasts in a CPU device with an ONNXRuntime environment."""
-
-
-markdown_batch_inference = """
-This chart shows how the model's speed changes with different batch sizes . As the batch size increases, you can observe an increase in FPS (images per second).
-"""

supervisely/nn/benchmark/visualization/vis_metric_base.py

@@ -1,337 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List, Optional
-
-if TYPE_CHECKING:
-    from supervisely.nn.benchmark.visualization.visualizer import Visualizer
-
-from jinja2 import Template
-
-from supervisely._utils import camel_to_snake
-from supervisely.nn.benchmark.cv_tasks import CVTask
-from supervisely.nn.benchmark.visualization.vis_templates import (
-    template_chart_str,
-    template_gallery_str,
-    template_markdown_str,
-    template_notification_str,
-    template_radiogroup_str,
-    template_table_str,
-)
-from supervisely.nn.benchmark.visualization.vis_widgets import Schema, Widget
-from supervisely.project.project_meta import ProjectMeta
-
-
-class MetricVis:
-
-    def __init__(self, loader: Visualizer) -> None:
-
-        self.cv_tasks: List[CVTask] = CVTask.values()
-        self.clickable: bool = False
-        self.has_diffs_view: bool = False
-        self.switchable: bool = False
-        self.schema: Schema = None
-        self.empty = False
-        self._is_overview = False
-
-        self._loader = loader
-        self._template_markdown = Template(template_markdown_str)
-        self._template_chart = Template(template_chart_str)
-        self._template_radiogroup = Template(template_radiogroup_str)
-        self._template_gallery = Template(template_gallery_str)
-        self._template_table = Template(template_table_str)
-        self._template_notification = Template(template_notification_str)
-        self._keypair_sep = "-"
-
-    @property
-    def radiogroup_id(self) -> Optional[str]:
-        if self.switchable:
-            return f"radiogroup_" + self.name
-
-    @property
-    def template_sidebar_str(self) -> str:
-        res = ""
-        for widget in self.schema:  # pylint: disable=not-an-iterable
-            if isinstance(widget, Widget.Markdown):
-                if widget.title is not None and widget.is_header:
-                    res += f"""\n          <div>\n            <el-button type="text" @click="data.scrollIntoView='{widget.id}'" """
-                    res += (
-                        """:style="{fontWeight: data.scrollIntoView==='"""
-                        + widget.id
-                        + """' ? 'bold' : 'normal'}"""
-                    )
-                    res += f""" ">{widget.title}</el-button>\n          </div>"""
-        return res
-
-    @property
-    def template_main_str(self) -> str:
-        res = ""
-
-        _is_before_chart = True
-
-        def _add_radio_buttons():
-            res = ""
-            for widget in self.schema:  # pylint: disable=not-an-iterable
-                if isinstance(widget, Widget.Chart):
-                    basename = f"{widget.name}_{self.name}"
-                    res += "\n            {{ " + f"el_radio_{basename}_html" + " }}"
-            return res
-
-        is_radiobuttons_added = False
-
-        for widget in self.schema:  # pylint: disable=not-an-iterable
-            if isinstance(widget, Widget.Chart):
-                _is_before_chart = False
-
-            if (
-                isinstance(widget, (Widget.Markdown, Widget.Notification, Widget.Collapse))
-                and _is_before_chart
-            ):
-                res += "\n            {{ " + f"{widget.name}_html" + " }}"
-                continue
-
-            if isinstance(widget, (Widget.Chart, Widget.Gallery, Widget.Table)):
-                basename = f"{widget.name}_{self.name}"
-                if self.switchable and not is_radiobuttons_added:
-                    res += _add_radio_buttons()
-                    is_radiobuttons_added = True
-                res += "\n            {{ " + f"{basename}_html" + " }}"
-                if self.clickable:
-                    res += "\n            {{ " + f"{basename}_clickdata_html" + " }}"
-                continue
-
-            if (
-                isinstance(widget, (Widget.Markdown, Widget.Notification, Widget.Collapse))
-                and not _is_before_chart
-            ):
-                res += "\n            {{ " + f"{widget.name}_html" + " }}"
-                continue
-
-        return res
-
-    def get_html_snippets(self) -> dict:
-        res = {}
-        for widget in self.schema:  # pylint: disable=not-an-iterable
-            if isinstance(widget, Widget.Markdown):
-                res[f"{widget.name}_html"] = self._template_markdown.render(
-                    {
-                        "widget_id": widget.id,
-                        "data_source": f"/data/{widget.name}.md",
-                        "command": "command",
-                        "data": "data",
-                        "is_overview": widget.title == "Overview",
-                    }
-                )
-
-            if isinstance(widget, Widget.Collapse):
-                subres = {}
-                for subwidget in widget.schema:
-                    if isinstance(subwidget, Widget.Markdown):
-                        subres[f"{subwidget.name}_html"] = self._template_markdown.render(
-                            {
-                                "widget_id": subwidget.id,
-                                "data_source": f"/data/{subwidget.name}.md",
-                                "command": "command",
-                                "data": "data",
-                            }
-                        )
-                res[f"{widget.name}_html"] = widget.template_schema.render(**subres)
-                continue
-
-            if isinstance(widget, Widget.Notification):
-                res[f"{widget.name}_html"] = self._template_notification.render(
-                    {
-                        "widget_id": widget.id,
-                        "data": "data",
-                        "title": widget.title.format(*widget.formats_title),
-                        "description": widget.description.format(*widget.formats_desc),
-                    }
-                )
-
-            if isinstance(widget, Widget.Chart):
-                basename = f"{widget.name}_{self.name}"
-                if self.switchable:
-                    res[f"el_radio_{basename}_html"] = self._template_radiogroup.render(
-                        {
-                            "radio_group": self.radiogroup_id,
-                            "switch_key": widget.switch_key,
-                        }
-                    )
-                chart_click_path = f"/data/{basename}_click_data.json" if self.clickable else None
-                chart_modal_data_source = f"/data/modal_general.json" if self.clickable else None
-                res[f"{basename}_html"] = self._template_chart.render(
-                    {
-                        "widget_id": widget.id,
-                        "init_data_source": f"/data/{basename}.json",
-                        "chart_click_data_source": chart_click_path,
-                        "command": "command",
-                        "data": "data",
-                        "cls_name": self.name,
-                        "key_separator": self._keypair_sep,
-                        "switchable": self.switchable,
-                        "radio_group": self.radiogroup_id,
-                        "switch_key": widget.switch_key,
-                        "chart_modal_data_source": chart_modal_data_source,
-                    }
-                )
-            if isinstance(widget, Widget.Gallery):
-                basename = f"{widget.name}_{self.name}"
-                if widget.is_table_gallery:
-                    for w in self.schema:  # pylint: disable=not-an-iterable
-                        if isinstance(w, Widget.Table):
-                            w.gallery_id = widget.id
-
-                gallery_click_data_source = (
-                    f"/data/{basename}_click_data.json" if self.clickable else None
-                )
-                gallery_modal_data_source = (
-                    f"/data/{basename}_modal_data.json" if self.clickable else None
-                )
-                gallery_diff_data_source = (
-                    f"/data/{basename}_diff_data.json" if self.has_diffs_view else None
-                )
-                res[f"{basename}_html"] = self._template_gallery.render(
-                    {
-                        "widget_id": widget.id,
-                        "init_data_source": f"/data/{basename}.json",
-                        "command": "command",
-                        "data": "data",
-                        "is_table_gallery": widget.is_table_gallery,
-                        "gallery_click_data_source": gallery_click_data_source,
-                        "gallery_diff_data_source": gallery_diff_data_source,
-                        "gallery_modal_data_source": gallery_modal_data_source,
-                    }
-                )
-
-            if isinstance(widget, Widget.Table):
-                basename = f"{widget.name}_{self.name}"
-                res[f"{basename}_html"] = self._template_table.render(
-                    {
-                        "widget_id": widget.id,
-                        "init_data_source": f"/data/{basename}.json",
-                        "command": "command",
-                        "data": "data",
-                        "table_click_data": f"/data/{widget.name}_{self.name}_click_data.json",
-                        "table_gallery_id": f"modal_general",
-                        "clickable": self.clickable,
-                        "mainColumn": widget.main_column,
-                        "fixColumns": widget.fixed_columns,
-                        "showHeaderControls": widget.show_header_controls,
-                        "width": f"width:{widget.width}" if widget.width else "",
-                    }
-                )
-
-        return res
-
-    @property
-    def name(self) -> str:
-        return camel_to_snake(self.__class__.__name__)
-
-    @property
-    def f1_optimal_conf(self) -> Optional[float]:
-        return self._loader.f1_optimal_conf
-
-    def get_figure(self, widget: Widget.Chart):  # -> Optional[go.Figure]:
-        pass
-
-    def get_click_data(self, widget: Widget.Chart) -> Optional[dict]:
-        if not self.clickable:
-            return
-
-        res = {}
-
-        res["layoutTemplate"] = [None, None, None]
-        res["clickData"] = {}
-        for key, v in self._loader.click_data.objects_by_class.items():
-            res["clickData"][key] = {}
-            res["clickData"][key]["imagesIds"] = []
-
-            # tmp = defaultdict(list)
-            img_ids = set()
-            obj_ids = set()
-
-            res["clickData"][key][
-                "title"
-            ] = f"{key} class: {len(v)} object{'s' if len(v) > 1 else ''}"
-
-            for x in v:
-                img_ids.add(x["dt_img_id"])
-                obj_ids.add(x["dt_obj_id"])
-
-            res["clickData"][key]["imagesIds"] = list(img_ids)
-            res["clickData"][key]["filters"] = [
-                {"type": "tag", "tagId": "confidence", "value": [self.f1_optimal_conf, 1]},
-                {"type": "tag", "tagId": "outcome", "value": "TP"},
-                {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
-            ]
-
-        return res
-
-    def get_modal_data(self, widget: Widget.Chart) -> Optional[dict]:
-        res = {}
-        api = self._loader._api
-        gt_project_id = self._loader.gt_project_info.id
-        dt_project_id = self._loader.dt_project_info.id
-        diff_project_id = self._loader.diff_project_info.id
-        gt_dataset = api.dataset.get_list(gt_project_id)[0]
-        dt_dataset = api.dataset.get_list(dt_project_id)[0]
-        diff_dataset = api.dataset.get_list(diff_project_id)[0]
-        gt_image_infos = api.image.get_list(dataset_id=gt_dataset.id)[:3]
-        pred_image_infos = api.image.get_list(dataset_id=dt_dataset.id)[:3]
-        diff_image_infos = api.image.get_list(dataset_id=diff_dataset.id)[:3]
-        project_metas = [
-            ProjectMeta.from_json(data=api.project.get_meta(id=x))
-            for x in [gt_project_id, dt_project_id, diff_project_id]
-        ]
-        for gt_image, pred_image, diff_image in zip(
-            gt_image_infos, pred_image_infos, diff_image_infos
-        ):
-            image_infos = [gt_image, pred_image, diff_image]
-            ann_infos = [api.annotation.download(x.id) for x in image_infos]
-
-            for idx, (image_info, ann_info, project_meta) in enumerate(
-                zip(image_infos, ann_infos, project_metas)
-            ):
-                image_name = image_info.name
-                image_url = image_info.preview_url
-                is_ignore = True if idx in [0, 1] else False
-                widget.gallery.append(
-                    title=image_name,
-                    image_url=image_url,
-                    annotation_info=ann_info,
-                    column_index=idx,
-                    project_meta=project_meta,
-                    ignore_tags_filtering=is_ignore,
-                )
-
-        res.update(widget.gallery.get_json_state())
-        res.update(widget.gallery.get_json_data()["content"])
-        res["layoutData"] = res.pop("annotations")
-        res["projectMeta"] = project_metas[0].to_json()
-
-        res.pop("layout")
-        res.pop("layoutData")
-
-        return res
-
-    def get_table(self, widget: Widget.Table) -> Optional[dict]:
-        pass
-
-    def get_gallery(self, widget: Widget.Gallery) -> Optional[dict]:
-        pass
-
-    def get_gallery_click_data(self, widget: Widget.Gallery) -> Optional[dict]:
-        pass
-
-    def get_diff_gallery_data(self, widget: Widget.Gallery) -> Optional[dict]:
-        pass
-
-    def get_md_content(self, widget: Widget.Markdown):
-        if hasattr(self._loader.vis_texts, widget.name):
-            return getattr(self._loader.vis_texts, widget.name).format(*widget.formats)
-        elif hasattr(self._loader.inference_speed_text, widget.name):
-            return getattr(self._loader.inference_speed_text, widget.name).format(*widget.formats)
-        else:
-            raise AttributeError(f"Not found texts template for {widget.name}")
-
-    def initialize_formats(self, loader: Visualizer, widget: Widget):
-        pass

supervisely/nn/benchmark/visualization/vis_metrics/__init__.py

@@ -1,67 +0,0 @@
-from supervisely.nn.benchmark.visualization.vis_metrics.confidence_distribution import (
-    ConfidenceDistribution,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.confidence_score import (
-    ConfidenceScore,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.confusion_matrix import (
-    ConfusionMatrix,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.explorer_grid import (
-    ExplorerGrid,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.f1_score_at_different_iou import (
-    F1ScoreAtDifferentIOU,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.frequently_confused import (
-    FrequentlyConfused,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.iou_distribution import (
-    IOUDistribution,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.model_predictions import (
-    ModelPredictions,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.outcome_counts import (
-    OutcomeCounts,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.outcome_counts_per_class import (
-    PerClassOutcomeCounts,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.overview import Overview
-from supervisely.nn.benchmark.visualization.vis_metrics.percision_avg_per_class import (
-    PerClassAvgPrecision,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.pr_curve import PRCurve
-from supervisely.nn.benchmark.visualization.vis_metrics.pr_curve_by_class import (
-    PRCurveByClass,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.precision import Precision
-from supervisely.nn.benchmark.visualization.vis_metrics.recall import Recall
-from supervisely.nn.benchmark.visualization.vis_metrics.recall_vs_precision import (
-    RecallVsPrecision,
-)
-from supervisely.nn.benchmark.visualization.vis_metrics.reliability_diagram import (
-    ReliabilityDiagram,
-)
-
-ALL_METRICS = (
-    Overview,
-    ExplorerGrid,
-    ModelPredictions,
-    OutcomeCounts,
-    Recall,
-    Precision,
-    RecallVsPrecision,
-    PRCurve,
-    PRCurveByClass,
-    ConfusionMatrix,
-    FrequentlyConfused,
-    IOUDistribution,
-    ReliabilityDiagram,
-    ConfidenceScore,
-    F1ScoreAtDifferentIOU,
-    ConfidenceDistribution,
-    PerClassAvgPrecision,
-    PerClassOutcomeCounts,
-)