supervisely 6.73.419__py3-none-any.whl → 6.73.421__py3-none-any.whl

supervisely/template/experiment/experiment_generator.py

@@ -1,16 +1,32 @@
 from __future__ import annotations
 
 import os
+import math
 from datetime import datetime
+import json
 from pathlib import Path
-from typing import Dict, Literal, Optional, Tuple
+from typing import Dict, Literal, Optional, Tuple, List
+from urllib.parse import urlencode
 
 import supervisely.io.env as sly_env
 import supervisely.io.fs as sly_fs
 import supervisely.io.json as sly_json
-from supervisely import logger
+from supervisely import logger, ProjectInfo
 from supervisely.api.api import Api
 from supervisely.api.file_api import FileInfo
+from supervisely.geometry.any_geometry import AnyGeometry
+from supervisely.geometry.bitmap import Bitmap
+from supervisely.geometry.cuboid import Cuboid
+from supervisely.geometry.cuboid_3d import Cuboid3d
+from supervisely.geometry.graph import GraphNodes
+from supervisely.geometry.multichannel_bitmap import MultichannelBitmap
+from supervisely.geometry.point import Point
+from supervisely.geometry.point_3d import Point3d
+from supervisely.geometry.pointcloud import Pointcloud
+from supervisely.geometry.polygon import Polygon
+from supervisely.geometry.polyline import Polyline
+from supervisely.geometry.rectangle import Rectangle
+from supervisely.imaging.color import rgb2hex
 from supervisely.nn.benchmark.object_detection.metric_provider import (
     METRIC_NAMES as OBJECT_DETECTION_METRIC_NAMES,
 )
@@ -23,21 +39,16 @@ from supervisely.nn.utils import RuntimeType
 from supervisely.project import ProjectMeta
 from supervisely.template.base_generator import BaseGenerator
 
-from supervisely.geometry.any_geometry import AnyGeometry
-from supervisely.geometry.cuboid_3d import Cuboid3d
-from supervisely.geometry.point_3d import Point3d
-from supervisely.geometry.pointcloud import Pointcloud
-from supervisely.geometry.multichannel_bitmap import MultichannelBitmap
-from supervisely.geometry.cuboid import Cuboid
-from supervisely.geometry.graph import GraphNodes
-from supervisely.geometry.bitmap import Bitmap
-from supervisely.geometry.point import Point
-from supervisely.geometry.polyline import Polyline
-from supervisely.geometry.polygon import Polygon
-from supervisely.geometry.rectangle import Rectangle
-
-from supervisely.imaging.color import rgb2hex
-
+try:
+    from tbparse import SummaryReader  # pylint: disable=import-error
+    import plotly.express as px  # pylint: disable=import-error
+    from plotly.subplots import make_subplots  # pylint: disable=import-error
+    import plotly.graph_objects as go  # pylint: disable=import-error
+except Exception as _:
+    SummaryReader = None  # type: ignore
+    px = None  # type: ignore
+    make_subplots = None  # type: ignore
+    go = None  # type: ignore
 
 # @TODO: Partly supports unreleased apps
 class ExperimentGenerator(BaseGenerator):
@@ -79,6 +90,12 @@ class ExperimentGenerator(BaseGenerator):
     def _report_url(self, server_address: str, template_id: int) -> str:
         return f"{server_address}/nn/experiments/{template_id}"
 
+    def _datasets_url_with_entities_filter(self, project_id: int, entities_filter: List[dict]) -> str:
+        base_url = self.api.server_address.rstrip('/')
+        path = f"/projects/{project_id}/datasets"
+        query = urlencode({"entitiesFilter": json.dumps(entities_filter)})
+        return f"{base_url}{path}?{query}"
+
     def upload_to_artifacts(self):
         remote_dir = os.path.join(self.info["artifacts_dir"], "visualization")
         self.upload(remote_dir, team_id=self.team_id)
@@ -87,9 +104,7 @@ class ExperimentGenerator(BaseGenerator):
         remote_report_path = os.path.join(
             self.info["artifacts_dir"], "visualization", "template.vue"
         )
-        experiment_report = self.api.file.get_info_by_path(
-            self.team_id, remote_report_path
-        )
+        experiment_report = self.api.file.get_info_by_path(self.team_id, remote_report_path)
         if experiment_report is None:
             raise ValueError("Generate and upload report first")
         return experiment_report
@@ -107,7 +122,7 @@ class ExperimentGenerator(BaseGenerator):
         context = {
             "env": self._get_env_context(),
             "experiment": self._get_experiment_context(),
-            "resources": self._get_links_context(),
+            "resources": self._get_resources_context(),
             "code": self._get_code_context(),
             "widgets": self._get_widgets_context(),
         }
@@ -119,18 +134,32 @@ class ExperimentGenerator(BaseGenerator):
 
     def _get_apps_context(self):
         train_app, serve_app = self._get_app_train_serve_app_info()
-        apply_images_app, apply_videos_app = self._get_app_apply_nn_app_info()
         log_viewer_app = self._get_log_viewer_app_info()
+        apply_images_app, apply_videos_app = self._get_app_apply_nn_app_info()
+        predict_app = self._get_predict_app_info()
         return {
             "train": train_app,
             "serve": serve_app,
             "log_viewer": log_viewer_app,
             "apply_nn_to_images": apply_images_app,
             "apply_nn_to_videos": apply_videos_app,
+            "predict": predict_app,
         }
 
-    def _get_links_context(self):
-        return {"apps": self._get_apps_context()}
+    def _get_original_repository_info(self):
+        original_repository = self.app_options.get("original_repository", None)
+        if original_repository is None:
+            return None
+        original_repository_info = {
+            "name": original_repository.get("name", None),
+            "url": original_repository.get("url", None),
+        }
+        return original_repository_info
+
+    def _get_resources_context(self):
+        apps = self._get_apps_context()
+        original_repository = self._get_original_repository_info()
+        return {"apps": apps, "original_repository": original_repository}
 
     def _get_code_context(self):
         docker_image = self._get_docker_image()
@@ -141,12 +170,8 @@ class ExperimentGenerator(BaseGenerator):
             "docker": {"image": docker_image, "deploy": f"{docker_image}-deploy"},
             "local_prediction": {
                 "repo": repo_info,
-                "serving_module": (
-                    self.serving_class.__module__ if self.serving_class else None
-                ),
-                "serving_class": (
-                    self.serving_class.__name__ if self.serving_class else None
-                ),
+                "serving_module": (self.serving_class.__module__ if self.serving_class else None),
+                "serving_class": (self.serving_class.__name__ if self.serving_class else None),
             },
             "demo": {
                 "pytorch": pytorch_demo,
@@ -160,7 +185,7 @@ class ExperimentGenerator(BaseGenerator):
         metrics_table = self._generate_metrics_table(self.info["task_type"])
         sample_gallery = self._get_sample_predictions_gallery()
         classes_table = self._generate_classes_table()
-
+        training_plots = self._generate_training_plots()
         return {
             "tables": {
                 "checkpoints": checkpoints_table,
@@ -168,6 +193,7 @@ class ExperimentGenerator(BaseGenerator):
                 "classes": classes_table,
             },
             "sample_pred_gallery": sample_gallery,
+            "training_plots": training_plots,
         }
 
     # --------------------------------------------------------------------------- #
@@ -186,10 +212,7 @@ class ExperimentGenerator(BaseGenerator):
         html.append("<thead><tr><th>Metrics</th><th>Value</th></tr></thead>")
         html.append("<tbody>")
 
-        if (
-            task_type == TaskType.OBJECT_DETECTION
-            or task_type == TaskType.INSTANCE_SEGMENTATION
-        ):
+        if task_type == TaskType.OBJECT_DETECTION or task_type == TaskType.INSTANCE_SEGMENTATION:
             metric_names = OBJECT_DETECTION_METRIC_NAMES
         elif task_type == TaskType.SEMANTIC_SEGMENTATION:
             metric_names = SEMANTIC_SEGMENTATION_METRIC_NAMES
@@ -313,9 +336,7 @@ class ExperimentGenerator(BaseGenerator):
         primary_metric_value = eval_metrics.get(primary_metric_name)
 
         if primary_metric_name is None or primary_metric_value is None:
-            logger.debug(
-                f"Primary metric is not found in evaluation metrics: {eval_metrics}"
-            )
+            logger.debug(f"Primary metric is not found in evaluation metrics: {eval_metrics}")
             return primary_metric
 
         primary_metric = {
@@ -339,10 +360,7 @@ class ExperimentGenerator(BaseGenerator):
             return display_metrics
 
         main_metrics = []
-        if (
-            task_type == TaskType.OBJECT_DETECTION
-            or task_type == TaskType.INSTANCE_SEGMENTATION
-        ):
+        if task_type == TaskType.OBJECT_DETECTION or task_type == TaskType.INSTANCE_SEGMENTATION:
             main_metrics = ["mAP", "AP75", "AP50", "precision", "recall"]
         elif task_type == TaskType.SEMANTIC_SEGMENTATION:
             main_metrics = ["mIoU", "mPixel", "mPrecision", "mRecall", "mF1"]
@@ -384,29 +402,20 @@ class ExperimentGenerator(BaseGenerator):
             if trt_checkpoint is not None:
                 checkpoints.append(trt_checkpoint)
 
-        checkpoint_paths = [
-            os.path.join(self.artifacts_dir, ckpt) for ckpt in checkpoints
-        ]
+        checkpoint_paths = [os.path.join(self.artifacts_dir, ckpt) for ckpt in checkpoints]
         checkpoint_infos = [
-            self.api.file.get_info_by_path(self.team_id, path)
-            for path in checkpoint_paths
-        ]
-        checkpoint_sizes = [
-            f"{info.sizeb / 1024 / 1024:.2f} MB" for info in checkpoint_infos
+            self.api.file.get_info_by_path(self.team_id, path) for path in checkpoint_paths
         ]
+        checkpoint_sizes = [f"{info.sizeb / 1024 / 1024:.2f} MB" for info in checkpoint_infos]
         checkpoint_dl_links = [
             f"<a href='{info.full_storage_url}' download='{sly_fs.get_file_name_with_ext(info.path)}'>Download</a>"
             for info in checkpoint_infos
         ]
 
         html = ['<table class="table">']
-        html.append(
-            "<thead><tr><th>Checkpoint</th><th>Size</th><th> </th></tr></thead>"
-        )
+        html.append("<thead><tr><th>Checkpoint</th><th>Size</th><th> </th></tr></thead>")
         html.append("<tbody>")
-        for checkpoint, size, dl_link in zip(
-            checkpoints, checkpoint_sizes, checkpoint_dl_links
-        ):
+        for checkpoint, size, dl_link in zip(checkpoints, checkpoint_sizes, checkpoint_dl_links):
             if isinstance(checkpoint, str):
                 html.append(
                     f"<tr><td>{os.path.basename(checkpoint)}</td><td>{size}</td><td>{dl_link}</td></tr>"
@@ -517,9 +526,7 @@ class ExperimentGenerator(BaseGenerator):
         best_checkpoint_path = os.path.join(
             self.artifacts_dir, "checkpoints", self.info["best_checkpoint"]
         )
-        best_checkpoint_info = self.api.file.get_info_by_path(
-            self.team_id, best_checkpoint_path
-        )
+        best_checkpoint_info = self.api.file.get_info_by_path(self.team_id, best_checkpoint_path)
         best_checkpoint = {
             "name": self.info["best_checkpoint"],
             "path": best_checkpoint_path,
@@ -560,9 +567,7 @@ class ExperimentGenerator(BaseGenerator):
                 self.team_id,
                 os.path.join(os.path.dirname(onnx_checkpoint_path), "classes.json"),
             )
-            onnx_file_info = self.api.file.get_info_by_path(
-                self.team_id, onnx_checkpoint_path
-            )
+            onnx_file_info = self.api.file.get_info_by_path(self.team_id, onnx_checkpoint_path)
             onnx_checkpoint_data = {
                 "name": os.path.basename(export.get(RuntimeType.ONNXRUNTIME)),
                 "path": onnx_checkpoint_path,
@@ -572,16 +577,12 @@ class ExperimentGenerator(BaseGenerator):
             }
         trt_checkpoint = export.get(RuntimeType.TENSORRT)
         if trt_checkpoint is not None:
-            trt_checkpoint_path = os.path.join(
-                self.artifacts_dir, export.get(RuntimeType.TENSORRT)
-            )
+            trt_checkpoint_path = os.path.join(self.artifacts_dir, export.get(RuntimeType.TENSORRT))
             classes_file = self.api.file.get_info_by_path(
                 self.team_id,
                 os.path.join(os.path.dirname(trt_checkpoint_path), "classes.json"),
             )
-            trt_file_info = self.api.file.get_info_by_path(
-                self.team_id, trt_checkpoint_path
-            )
+            trt_file_info = self.api.file.get_info_by_path(self.team_id, trt_checkpoint_path)
             trt_checkpoint_data = {
                 "name": os.path.basename(export.get(RuntimeType.TENSORRT)),
                 "path": trt_checkpoint_path,
@@ -638,9 +639,7 @@ class ExperimentGenerator(BaseGenerator):
             root_dir = current_dir
 
             while root_dir.parent != root_dir:
-                config_path = (
-                    root_dir / "supervisely_integration" / "train" / "config.json"
-                )
+                config_path = root_dir / "supervisely_integration" / "train" / "config.json"
                 if config_path.exists():
                     break
                 root_dir = root_dir.parent
@@ -748,9 +747,7 @@ class ExperimentGenerator(BaseGenerator):
         :rtype: Tuple[str, str]
         """
 
-        def find_app_by_framework(
-            api: Api, framework: str, action: Literal["train", "serve"]
-        ):
+        def find_app_by_framework(api: Api, framework: str, action: Literal["train", "serve"]):
             try:
                 modules = api.app.get_list_ecosystem_modules(
                     categories=[action, f"framework:{framework}"],
@@ -763,26 +760,18 @@ class ExperimentGenerator(BaseGenerator):
                 logger.warning(f"Failed to find {action} app by framework: {e}")
                 return None
 
-        train_app_info = find_app_by_framework(
-            self.api, self.info["framework_name"], "train"
-        )
-        serve_app_info = find_app_by_framework(
-            self.api, self.info["framework_name"], "serve"
-        )
+        train_app_info = find_app_by_framework(self.api, self.info["framework_name"], "train")
+        serve_app_info = find_app_by_framework(self.api, self.info["framework_name"], "serve")
 
         if train_app_info is not None:
-            train_app_slug = train_app_info["slug"].replace(
-                "supervisely-ecosystem/", ""
-            )
+            train_app_slug = train_app_info["slug"].replace("supervisely-ecosystem/", "")
             train_app_id = train_app_info["id"]
         else:
             train_app_slug = None
             train_app_id = None
 
         if serve_app_info is not None:
-            serve_app_slug = serve_app_info["slug"].replace(
-                "supervisely-ecosystem/", ""
-            )
+            serve_app_slug = serve_app_info["slug"].replace("supervisely-ecosystem/", "")
             serve_app_id = serve_app_info["id"]
         else:
             serve_app_slug = None
@@ -814,9 +803,7 @@ class ExperimentGenerator(BaseGenerator):
         if task_info is not None:
             agent_info["name"] = task_info["agentName"]
             agent_info["id"] = task_info["agentId"]
-            agent_info["link"] = (
-                f"{self.api.server_address}/nodes/{agent_info['id']}/info"
-            )
+            agent_info["link"] = f"{self.api.server_address}/nodes/{agent_info['id']}/info"
         return agent_info
 
     def _get_class_names(self, model_classes: list) -> dict:
@@ -839,51 +826,111 @@ class ExperimentGenerator(BaseGenerator):
             ),
         }
 
+    def _get_predict_app_info(self):
+        """
+        Get predict app info.
+
+        :returns: Predict app info
+        :rtype: dict
+        """
+        predict_app_slug = "supervisely-ecosystem/apply-nn"
+        predict_app_module_id = self.api.app.get_ecosystem_module_id(predict_app_slug)
+        predict_app = {"slug": predict_app_slug, "module_id": predict_app_module_id}
+        return predict_app
+
     def _get_app_apply_nn_app_info(self):
-        """Get apply NN app info.
+        """
+        Get apply NN app info.
 
         :returns: Apply NN app info
         :rtype: dict
         """
-
+        # Images
         apply_nn_images_slug = "nn-image-labeling/project-dataset"
         apply_nn_images_module_id = self.api.app.get_ecosystem_module_id(
             f"supervisely-ecosystem/{apply_nn_images_slug}"
         )
+        apply_nn_images_app = {"slug": apply_nn_images_slug, "module_id": apply_nn_images_module_id}
+
+        # Videos
         apply_nn_videos_slug = "apply-nn-to-videos-project"
         apply_nn_videos_module_id = self.api.app.get_ecosystem_module_id(
             f"supervisely-ecosystem/{apply_nn_videos_slug}"
         )
+        apply_nn_videos_app = {"slug": apply_nn_videos_slug, "module_id": apply_nn_videos_module_id}
+        return apply_nn_images_app, apply_nn_videos_app
 
-        apply_nn_images_app = {
-            "slug": apply_nn_images_slug,
-            "module_id": apply_nn_images_module_id,
-        }
-        apply_nn_videos_app = {
-            "slug": apply_nn_videos_slug,
-            "module_id": apply_nn_videos_module_id,
+    def _get_project_splits(self):
+        train_collection_id = self.info.get("train_collection_id", None)
+        if train_collection_id is not None:
+            train_collection_info = self.api.entities_collection.get_info_by_id(train_collection_id)
+            train_collection_name = train_collection_info.name
+            train_collection_url = self._datasets_url_with_entities_filter(self.info["project_id"], [{"type": "entities_collection", "data": {"collectionId": train_collection_id, "include": True}}])
+            train_size = self.info.get("train_size", "N/A")
+        else:
+            train_collection_name = None
+            train_size = None
+            train_collection_url = None
+
+        val_collection_id = self.info.get("val_collection_id", None)
+        if val_collection_id is not None:
+            val_collection_info = self.api.entities_collection.get_info_by_id(val_collection_id)
+            val_collection_name = val_collection_info.name
+            val_collection_url = self._datasets_url_with_entities_filter(self.info["project_id"], [{"type": "entities_collection", "data": {"collectionId": val_collection_id, "include": True}}])
+            val_size = self.info.get("val_size", "N/A")
+        else:
+            val_collection_name = None
+            val_size = None
+            val_collection_url = None
+
+        splits = {
+            "train": {
+                "name": train_collection_name,
+                "size": train_size,
+                "url": train_collection_url,
+            },
+            "val": {
+                "name": val_collection_name,
+                "size": val_size,
+                "url": val_collection_url,
+            },
         }
-        return apply_nn_images_app, apply_nn_videos_app
+        return splits
+
+    def _get_project_version(self, project_info: ProjectInfo):
+        project_version = project_info.version
+        if project_version is None:
+            version_info = {
+                "version": "N/A",
+                "id": None,
+                "url": None,
+            }
+        else:
+            version_info = {
+                "version": project_version["version"],
+                "id": project_version["id"],
+                "url": f"{self.api.server_address}/projects/{project_info.id}/versions",
+            }
+        return version_info
 
     def _get_project_context(self):
         project_id = self.info["project_id"]
         project_info = self.api.project.get_info_by_id(project_id)
+        project_version = self._get_project_version(project_info)
         project_type = project_info.type
         project_url = f"{self.api.server_address}/projects/{project_id}/datasets"
-        project_train_size = self.info.get("train_size", "N/A")
-        project_val_size = self.info.get("val_size", "N/A")
         model_classes = [cls.name for cls in self.model_meta.obj_classes]
         class_names = self._get_class_names(model_classes)
+        splits = self._get_project_splits()
 
         project_context = {
             "id": project_id,
+            "workspace_id": project_info.workspace_id if project_info else None,
             "name": project_info.name if project_info else "Project was archived",
-            "url": project_url,
-            "type": project_type,
-            "splits": {
-                "train": project_train_size,
-                "val": project_val_size,
-            },
+            "version": project_version,
+            "url": project_url if project_info else None,
+            "type": project_type if project_info else None,
+            "splits": splits,
             "classes": {
                 "count": len(model_classes),
                 "names": class_names,
@@ -897,10 +944,14 @@ class ExperimentGenerator(BaseGenerator):
         base_checkpoint_path = None
         if base_checkpoint_link is not None:
             if base_checkpoint_link.startswith("/experiments/"):
-                base_checkpoint_info = self.api.file.get_info_by_path(self.team_id, base_checkpoint_link)
+                base_checkpoint_info = self.api.file.get_info_by_path(
+                    self.team_id, base_checkpoint_link
+                )
                 base_checkpoint_name = base_checkpoint_info.name
                 base_checkpoint_link = base_checkpoint_info.full_storage_url
-                base_checkpoint_path = f"{self.api.server_address}/files/?path={base_checkpoint_info.path}"
+                base_checkpoint_path = (
+                    f"{self.api.server_address}/files/?path={base_checkpoint_info.path}"
+                )
 
         base_checkpoint = {
             "name": base_checkpoint_name,
@@ -930,9 +981,7 @@ class ExperimentGenerator(BaseGenerator):
         onnx_checkpoint, trt_checkpoint = self._get_optimized_checkpoints()
 
         logs_path = self.info.get("logs", {}).get("link")
-        logs_url = (
-            f"{self.api.server_address}/files/?path={logs_path}" if logs_path else None
-        )
+        logs_url = f"{self.api.server_address}/files/?path={logs_path}" if logs_path else None
 
         primary_metric = self._get_primary_metric()
         display_metrics = self._get_display_metrics(self.info["task_type"])
@@ -994,3 +1043,100 @@ class ExperimentGenerator(BaseGenerator):
             },
         }
         return experiment_context
+
+    def _generate_training_plots(self) -> Optional[str]:
+        # pip install tbparse plotly kaleido
+        if SummaryReader is None or px is None:
+            logger.warning("tbparse or plotly is not installed – skipping training plots generation")
+            return None
+
+        logs_path = self.info.get("logs", {}).get("link")
+        if logs_path is None:
+            return None
+
+        events_files: List[str] = []
+        remote_log_files = self.api.file.list(self.team_id, logs_path, return_type="fileinfo")
+        try:
+            for f in remote_log_files:
+                if f.name.startswith("events.out.tfevents"):
+                    events_files.append(f.path)
+        except Exception as e:
+            logger.warning(f"Failed to get training logs: {e}")
+            return None
+
+        if len(events_files) == 0:
+            return None
+
+        tmp_logs_dir = os.path.join(self.output_dir, "logs_tmp")
+        sly_fs.mkdir(tmp_logs_dir, True)
+        local_event_path = os.path.join(tmp_logs_dir, os.path.basename(events_files[0]))
+        try:
+            self.api.file.download(self.team_id, events_files[0], local_event_path)
+        except Exception as e:
+            logger.warning(f"Failed to download training log: {e}")
+            return None
+
+        try:
+            reader = SummaryReader(local_event_path)
+            scalars_df = reader.scalars
+        except Exception as e:
+            logger.warning(f"Failed to read training log: {e}")
+            return None
+
+        if scalars_df is None or scalars_df.empty:
+            return None
+
+        tags_to_plot = scalars_df["tag"].unique().tolist()[:12]
+        df_plot = scalars_df[scalars_df["tag"].isin(tags_to_plot)]
+
+        try:
+            data_dir = os.path.join(self.output_dir, "data")
+            if not sly_fs.dir_exists(data_dir):
+                sly_fs.mkdir(data_dir, True)
+
+            n_tags = len(tags_to_plot)
+            side = min(4, max(2, math.ceil(math.sqrt(n_tags))))
+            cols = side
+            rows = math.ceil(n_tags / cols)
+            fig = make_subplots(rows=rows, cols=cols, subplot_titles=tags_to_plot)
+
+            for idx, tag in enumerate(tags_to_plot, start=1):
+                tag_df = df_plot[df_plot["tag"] == tag]
+                if tag_df.empty:
+                    continue
+                row = (idx - 1) // cols + 1
+                col = (idx - 1) % cols + 1
+                fig.add_trace(
+                    go.Scatter(
+                        x=tag_df["step"],
+                        y=tag_df["value"],
+                        mode="lines",
+                        name=tag,
+                        showlegend=False,
+                    ),
+                    row=row,
+                    col=col,
+                )
+
+                if tag.startswith("lr"):
+                    fig.update_yaxes(tickformat=".0e", row=row, col=col)
+
+            fig.update_layout(
+                height=300 * rows,
+                width=400 * cols,
+                showlegend=False,
+            )
+
+            local_img_path = os.path.join(data_dir, "training_plots_grid.png")
+            fig.write_image(local_img_path, engine="kaleido")
+            sly_fs.remove_dir(tmp_logs_dir)
+
+            img_widget = f"<sly-iw-image src=\"/data/training_plots_grid.png\" :template-base-path=\"templateBasePath\" :options=\"{{ style: {{ width: '70%', height: 'auto' }} }}\" />"
+            return img_widget
+        except Exception as e:
+            logger.warning(f"Failed to build or save static training plot: {e}")
+            return None
+
+
+# Pylint Errors: ************* Module supervisely.api.app_api
+# supervisely/api/app_api.py:1463:20: E0606: Possibly using variable 'progress' before assignment (possibly-used-before-assignment)