supervisely 6.73.369__tar.gz → 6.73.370__tar.gz

{supervisely-6.73.369/supervisely.egg-info → supervisely-6.73.370}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.369
+Version: 6.73.370
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.369 → supervisely-6.73.370}/supervisely/app/widgets/experiment_selector/experiment_selector.py

@@ -52,7 +52,7 @@ class ExperimentSelector(Widget):
             self._experiment_info = experiment_info
 
             task_id = experiment_info.task_id
-            if task_id == "debug-session":
+            if task_id == "debug-session" or task_id == -1:
                 pass
             elif type(task_id) is str:
                 if task_id.isdigit():
@@ -392,7 +392,7 @@ class ExperimentSelector(Widget):
                 if result:
                     task_type, model_row = result
                     if task_type is not None and model_row is not None:
-                        if model_row.task_id == "debug-session":
+                        if model_row.task_id == "debug-session" or model_row.task_id == -1:
                             self.__debug_row = (task_type, model_row)
                             continue
                         table_rows[task_type].append(model_row)

{supervisely-6.73.369 → supervisely-6.73.370}/supervisely/nn/inference/inference.py

@@ -878,10 +878,16 @@ class Inference:
 
         try:
             if is_production():
-                self._add_workflow_input(model_source, model_files, model_info)
+                without_workflow = deploy_params.get("without_workflow", False)
+                if without_workflow is False:
+                    self._add_workflow_input(model_source, model_files, model_info)
         except Exception as e:
             logger.warning(f"Failed to add input to the workflow: {repr(e)}")
 
+        # remove is_benchmark from deploy_params
+        if "without_workflow" in deploy_params:
+            deploy_params.pop("without_workflow")
+
         self._load_model(deploy_params)
         if self._model_meta is None:
             self._set_model_meta_from_classes()
@@ -3709,17 +3715,20 @@ class Inference:
             )
 
         app_name = sly_env.app_name()
-        meta = WorkflowMeta(node_settings=WorkflowSettings(title=f"Serve {app_name}"))
+        meta = WorkflowMeta(node_settings=WorkflowSettings(title=app_name))
 
         logger.debug(
             f"Workflow Input: Checkpoint URL - {checkpoint_url}, Checkpoint Name - {checkpoint_name}"
         )
-        if checkpoint_url and self.api.file.exists(sly_env.team_id(), checkpoint_url):
-            self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta)
-        else:
-            logger.debug(
-                f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input"
-            )
+        if model_source == ModelSource.CUSTOM:
+            if checkpoint_url and self.api.file.exists(sly_env.team_id(), checkpoint_url):
+                # self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta)
+                remote_checkpoint_dir = os.path.dirname(checkpoint_url)
+                self.api.app.workflow.add_input_folder(remote_checkpoint_dir, meta=meta)
+            else:
+                logger.debug(
+                    f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input"
+                )
 
 
 def _exclude_duplicated_predictions(

{supervisely-6.73.369 → supervisely-6.73.370}/supervisely/nn/training/gui/gui.py

@@ -242,7 +242,7 @@ class TrainGUI:
         else:
             self.task_id = sly_env.task_id(raise_not_found=False)
             if self.task_id is None:
-                self.task_id = "debug-session"
+                self.task_id = -1
 
         self.framework_name = framework_name
         self.models = models
@@ -257,7 +257,7 @@ class TrainGUI:
         self.project_info = self._api.project.get_info_by_id(self.project_id)
         if self.project_info.type is None:
             raise ValueError(f"Project with ID: '{self.project_id}' does not exist or was archived")
-        
+
         self.project_meta = ProjectMeta.from_json(self._api.project.get_meta(self.project_id))
 
         if self.workspace_id is None:
@@ -350,7 +350,10 @@ class TrainGUI:
             if model_name is None:
                 experiment_name = "Enter experiment name"
             else:
-                experiment_name = f"{self.task_id}_{self.project_info.name}_{model_name}"
+                if self.task_id == -1:
+                    experiment_name = f"debug_{self.project_info.name}_{model_name}"
+                else:
+                    experiment_name = f"{self.task_id}_{self.project_info.name}_{model_name}"
 
             if experiment_name == self.training_process.get_experiment_name():
                 return

{supervisely-6.73.369 → supervisely-6.73.370}/supervisely/nn/training/train_app.py

@@ -9,7 +9,7 @@ import shutil
 import subprocess
 from datetime import datetime
 from os import getcwd, listdir, walk
-from os.path import basename, exists, expanduser, isdir, isfile, join
+from os.path import basename, dirname, exists, expanduser, isdir, isfile, join
 from typing import Any, Dict, List, Literal, Optional, Union
 from urllib.request import urlopen
 
@@ -128,7 +128,7 @@ class TrainApp:
                 self._app_name = "custom-app"
             self.task_id = sly_env.task_id(raise_not_found=False)
             if self.task_id is None:
-                self.task_id = "debug-session"
+                self.task_id = -1
             logger.info("TrainApp is running in debug mode")
 
         self.framework_name = framework_name
@@ -580,7 +580,7 @@ class TrainApp:
 
         # Step 6. Upload artifacts
         self._set_text_status("uploading")
-        remote_dir, file_info = self._upload_artifacts()
+        remote_dir, session_link_file_info = self._upload_artifacts()
 
         # Step 7. [Optional] Run Model Benchmark
         mb_eval_lnk_file_info, mb_eval_report = None, None
@@ -650,12 +650,35 @@ class TrainApp:
 
         # Step 10. Set output widgets
         self._set_text_status("reset")
-        self._set_training_output(experiment_info, remote_dir, file_info, mb_eval_report)
+        self._set_training_output(
+            experiment_info, remote_dir, session_link_file_info, mb_eval_report
+        )
         self._set_ws_progress_status("completed")
 
         # Step 11. Workflow output
         if is_production():
-            self._workflow_output(remote_dir, file_info, mb_eval_lnk_file_info, mb_eval_report_id)
+            best_checkpoint_file_info = self._get_best_checkpoint_info(experiment_info, remote_dir)
+            self._workflow_output(
+                remote_dir, best_checkpoint_file_info, mb_eval_lnk_file_info, mb_eval_report_id
+            )
+
+    def _get_best_checkpoint_info(self, experiment_info: dict, remote_dir: str) -> FileInfo:
+        """
+        Returns the best checkpoint info.
+
+        :param experiment_info: Experiment info.
+        :type experiment_info: dict
+        :param remote_dir: Remote directory.
+        :type remote_dir: str
+        :return: Best checkpoint info.
+        :rtype: FileInfo
+        """
+        best_checkpoint_name = experiment_info.get("best_checkpoint")
+        remote_best_checkpoint_path = join(remote_dir, "checkpoints", best_checkpoint_name)
+        best_checkpoint_file_info = self._api.file.get_info_by_path(
+            self.team_id, remote_best_checkpoint_path
+        )
+        return best_checkpoint_file_info
 
     def register_inference_class(
         self, inference_class: Inference, inference_settings: Union[str, dict] = None
@@ -686,6 +709,8 @@ class TrainApp:
         """
         Returns the current state of the application.
 
+        :param experiment_info: Experiment info.
+        :type experiment_info: dict
         :return: Application state.
         :rtype: dict
         """
@@ -748,6 +773,27 @@ class TrainApp:
         """
         self.gui.load_from_app_state(app_state)
 
+    def add_output_files(self, paths: List[str]) -> None:
+        """
+        Copies files or directories to the output directory, which will be uploaded to the team files upon training completion.
+        If path is a file, it will be uploaded to the root artifacts directory.
+        If path is a directory, it will be uploded to the root artifacts directory with the same directory name and structure.
+
+        :param paths: List of paths to files or directories to be copied to the output directory.
+        :type paths: List[str]
+        :return: None
+        :rtype: None
+        """
+
+        for path in paths:
+            if sly_fs.file_exists(path):
+                shutil.copyfile(path, join(self.output_dir, sly_fs.get_file_name_with_ext(path)))
+            elif sly_fs.dir_exists(path):
+                shutil.copytree(path, join(self.output_dir, basename(path)))
+            else:
+                logger.warning(f"Provided path: '{path}' does not exist. Skipping...")
+                continue
+
     # Loaders
     def _load_models(self, models: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
         """
@@ -1549,7 +1595,7 @@ class TrainApp:
         logger.debug(f"Uploading '{local_path}' to Supervisely")
         total_size = sly_fs.get_file_size(local_path)
         with self.progress_bar_main(
-            message=message, total=total_size, unit="bytes", unit_scale=True
+            message=message, total=total_size, unit="B", unit_scale=True, unit_divisor=1024
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
             file_info = self._api.file.upload(
@@ -1774,8 +1820,9 @@ class TrainApp:
         with self.progress_bar_main(
             message="Uploading demo files to Team Files",
             total=total_size,
-            unit="bytes",
+            unit="B",
             unit_scale=True,
+            unit_divisor=1024,
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
             remote_dir = self._api.file.upload_directory_fast(
@@ -1863,7 +1910,7 @@ class TrainApp:
             f"Uploading artifacts directory: '{self.output_dir}' to Supervisely Team Files directory '{remote_artifacts_dir}'"
         )
         # Clean debug directory if exists
-        if task_id == "debug-session":
+        if task_id == -1:
             if self._api.file.dir_exists(self.team_id, f"{remote_artifacts_dir}/", True):
                 with self.progress_bar_main(
                     message=f"[Debug] Cleaning train artifacts: '{remote_artifacts_dir}/'",
@@ -1888,8 +1935,9 @@ class TrainApp:
         with self.progress_bar_main(
             message="Uploading train artifacts to Team Files",
             total=total_size,
-            unit="bytes",
+            unit="B",
             unit_scale=True,
+            unit_divisor=1024,
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
             remote_dir = self._api.file.upload_directory_fast(
@@ -1907,6 +1955,7 @@ class TrainApp:
                 "state": {"slyFolder": f"{join(remote_dir, 'logs')}"}
             }
             self.gui.training_logs.tensorboard_offline_button.enable()
+
         return remote_dir, file_info
 
     def _set_training_output(
@@ -2095,6 +2144,7 @@ class TrainApp:
                 "model_meta": model_meta.to_json(),
                 "task_type": task_type,
             }
+            self._benchmark_params["without_workflow"] = True
 
             logger.info(f"Deploy parameters: {self._benchmark_params}")
 
@@ -2162,16 +2212,19 @@ class TrainApp:
                 raise ValueError(f"Task type: '{task_type}' is not supported for Model Benchmark")
 
             if self._has_splits_selector:
+                app_session_id = self.task_id
+                if app_session_id == -1:
+                    app_session_id = None
                 if self.gui.train_val_splits_selector.get_split_method() == "Based on datasets":
                     train_info = {
-                        "app_session_id": self.task_id,
+                        "app_session_id": app_session_id,
                         "train_dataset_ids": train_dataset_ids,
                         "train_images_ids": None,
                         "images_count": len(self._train_split),
                     }
                 else:
                     train_info = {
-                        "app_session_id": self.task_id,
+                        "app_session_id": app_session_id,
                         "train_dataset_ids": None,
                         "train_images_ids": train_images_ids,
                         "images_count": len(self._train_split),
@@ -2223,6 +2276,25 @@ class TrainApp:
 
         except Exception as e:
             logger.error(f"Model benchmark failed. {repr(e)}", exc_info=True)
+            pred_error_message = (
+                "Not found any predictions. Please make sure that your model produces predictions."
+            )
+            if isinstance(e, ValueError) and str(e) == pred_error_message:
+                self.gui.training_artifacts.model_benchmark_fail_text.set(
+                    "The Model Evaluation report cannot be generated: The model is not making predictions. "
+                    "This indicates that your model may not have trained successfully or is underfitted. "
+                    "You can try increasing the number of epochs or adjusting the hyperparameters more carefully.",
+                    "warning",
+                )
+
+            lnk_file_info, report, report_id, eval_metrics, primary_metric_name = (
+                None,
+                None,
+                None,
+                {},
+                None,
+            )
+
             self._set_text_status("finalizing")
             self.progress_bar_main.hide()
             self.progress_bar_secondary.hide()
@@ -2288,6 +2360,14 @@ class TrainApp:
     ):
         """
         Adds the output data to the workflow.
+
+        :param team_files_dir: Team files directory.
+        :type team_files_dir: str
+        :param file_info: FileInfo of the best checkpoint.
+        :type file_info: FileInfo
+        :param model_benchmark_report: FileInfo of the model benchmark report link (.lnk).
+        :type model_benchmark_report: Optional[FileInfo]
+        :param model_benchmark_report_id: Model benchmark report ID.
         """
         try:
             module_id = (
@@ -2310,7 +2390,7 @@ class TrainApp:
 
             if file_info:
                 relation_settings = WorkflowSettings(
-                    title="Train Artifacts",
+                    title="Checkpoints",
                     icon="folder",
                     icon_color="#FFA500",
                     icon_bg_color="#FFE8BE",
@@ -2321,7 +2401,10 @@ class TrainApp:
                     relation_settings=relation_settings, node_settings=node_settings
                 )
                 logger.debug(f"Workflow Output: meta \n    {meta}")
-                self._api.app.workflow.add_output_file(file_info, model_weight=True, meta=meta)
+                # self._api.app.workflow.add_output_file(file_info, model_weight=True, meta=meta)
+
+                remote_checkpoint_dir = dirname(file_info.path)
+                self._api.app.workflow.add_output_folder(remote_checkpoint_dir, meta=meta)
             else:
                 logger.debug(
                     f"File with checkpoints not found in Team Files. Cannot set workflow output."
@@ -2370,13 +2453,14 @@ class TrainApp:
             logger.debug("Tensorboard server is already running")
             return
         self._register_routes()
+
         args = [
             "tensorboard",
             "--logdir",
             self.log_dir,
             "--host=localhost",
             f"--port={self._tensorboard_port}",
-            "--load_fast=true",
+            "--load_fast=auto",
             "--reload_multifile=true",
         ]
         self._tensorboard_process = subprocess.Popen(args)
@@ -2522,6 +2606,9 @@ class TrainApp:
             self._set_ws_progress_status("finalizing")
             self._finalize(experiment_info)
             self.gui.training_process.start_button.loading = False
+
+            # Shutdown the app after training is finished
+            self.app.shutdown()
         except Exception as e:
             message = f"Error occurred during finalizing and uploading training artifacts. {check_logs_text}"
             self._show_error(message, e)
@@ -2706,6 +2793,7 @@ class TrainApp:
             total=size,
             unit="B",
             unit_scale=True,
+            unit_divisor=1024,
         ) as export_upload_main_pbar:
             logger.debug(f"Uploading {len(export_weights)} export weights of size {size} bytes")
             logger.debug(f"Destination paths: {file_dest_paths}")

{supervisely-6.73.369 → supervisely-6.73.370/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.369
+Version: 6.73.370
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely