supervisely 6.73.278__tar.gz → 6.73.279__tar.gz

{supervisely-6.73.278/supervisely.egg-info → supervisely-6.73.279}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.278
+Version: 6.73.279
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.278 → supervisely-6.73.279}/supervisely/nn/training/gui/gui.py

@@ -11,6 +11,9 @@ import supervisely.io.env as sly_env
 from supervisely import Api, ProjectMeta
 from supervisely._utils import is_production
 from supervisely.app.widgets import Stepper, Widget
+from supervisely.geometry.polygon import Polygon
+from supervisely.geometry.rectangle import Rectangle
+from supervisely.nn.task_type import TaskType
 from supervisely.nn.training.gui.classes_selector import ClassesSelector
 from supervisely.nn.training.gui.hyperparameters_selector import HyperparametersSelector
 from supervisely.nn.training.gui.input_selector import InputSelector
@@ -137,6 +140,26 @@ class TrainGUI:
                 return
             self.training_process.set_experiment_name(experiment_name)
 
+        def need_convert_class_shapes() -> bool:
+            task_type = self.model_selector.get_selected_task_type()
+
+            def _need_convert(shape):
+                if task_type == TaskType.OBJECT_DETECTION:
+                    return shape != Rectangle.geometry_name()
+                elif task_type in [TaskType.INSTANCE_SEGMENTATION, TaskType.SEMANTIC_SEGMENTATION]:
+                    return shape == Polygon.geometry_name()
+                return
+
+            data = self.classes_selector.classes_table._table_data
+            selected_classes = set(self.classes_selector.classes_table.get_selected_classes())
+            empty = set(r[0]["data"] for r in data if r[2]["data"] == 0 and r[3]["data"] == 0)
+            need_convert = set(r[0]["data"] for r in data if _need_convert(r[1]["data"]))
+
+            if need_convert.intersection(selected_classes - empty):
+                self.hyperparameters_selector.model_benchmark_auto_convert_warning.show()
+            else:
+                self.hyperparameters_selector.model_benchmark_auto_convert_warning.hide()
+
         # ------------------------------------------------- #
 
         # Wrappers
@@ -168,7 +191,7 @@ class TrainGUI:
             callback=self.hyperparameters_selector_cb,
             validation_text=self.model_selector.validator_text,
             validation_func=self.model_selector.validate_step,
-            on_select_click=[set_experiment_name],
+            on_select_click=[set_experiment_name, need_convert_class_shapes],
             collapse_card=(self.model_selector.card, self.collapsable),
         )
 
@@ -276,6 +299,7 @@ class TrainGUI:
         @self.hyperparameters_selector.run_model_benchmark_checkbox.value_changed
         def show_mb_speedtest(is_checked: bool):
             self.hyperparameters_selector.toggle_mb_speedtest(is_checked)
+            need_convert_class_shapes()
 
         # ------------------------------------------------- #
 

{supervisely-6.73.278 → supervisely-6.73.279}/supervisely/nn/training/gui/hyperparameters_selector.py

@@ -44,8 +44,18 @@ class HyperparametersSelector:
             self.model_benchmark_learn_more = Text(
                 f"Learn more about Model Benchmark in the {docs_link}.", status="info"
             )
+            self.model_benchmark_auto_convert_warning = Text(
+                text="Project will be automatically converted according to CV task for Model Evaluation.",
+                status="warning",
+            )
+            self.model_benchmark_auto_convert_warning.hide()
+
             self.display_widgets.extend(
-                [self.model_benchmark_field, self.model_benchmark_learn_more]
+                [
+                    self.model_benchmark_field,
+                    self.model_benchmark_learn_more,
+                    self.model_benchmark_auto_convert_warning,
+                ]
             )
         # -------------------------------- #
 

{supervisely-6.73.278 → supervisely-6.73.279}/supervisely/nn/training/gui/model_selector.py

@@ -12,9 +12,9 @@ from supervisely.app.widgets import (
     RadioTabs,
     Text,
 )
+from supervisely.nn.artifacts.utils import FrameworkMapper
 from supervisely.nn.experiments import get_experiment_infos
 from supervisely.nn.utils import ModelSource
-from supervisely.nn.artifacts.utils import FrameworkMapper, FrameworkName
 
 
 class ModelSelector:
@@ -37,7 +37,7 @@ class ModelSelector:
                 legacy_experiment_infos = framework_cls.get_list_experiment_info()
                 experiment_infos = experiment_infos + legacy_experiment_infos
             except:
-                logger.warn(f"Legacy checkpoints are not available for '{framework}'")
+                logger.warning(f"Legacy checkpoints are not available for '{framework}'")
 
         self.experiment_selector = ExperimentSelector(self.team_id, experiment_infos)
         self.model_source_tabs = RadioTabs(
@@ -106,3 +106,9 @@ class ModelSelector:
             self.validator_text.set(text="Model is selected", status="success")
             self.validator_text.show()
             return True
+
+    def get_selected_task_type(self) -> str:
+        if self.get_model_source() == ModelSource.PRETRAINED:
+            return self.pretrained_models_table.get_selected_task_type()
+        else:
+            return self.experiment_selector.get_selected_task_type()

{supervisely-6.73.278 → supervisely-6.73.279}/supervisely/nn/training/train_app.py

@@ -25,6 +25,7 @@ import supervisely.io.json as sly_json
 from supervisely import (
     Api,
     Application,
+    Dataset,
     DatasetInfo,
     OpenMode,
     Project,
@@ -32,6 +33,7 @@ from supervisely import (
     ProjectMeta,
     WorkflowMeta,
     WorkflowSettings,
+    batched,
     download_project,
     is_development,
     is_production,
@@ -340,22 +342,6 @@ class TrainApp:
         """
         return self.gui.model_selector.get_model_info()
 
-    @property
-    def model_meta(self) -> ProjectMeta:
-        """
-        Returns the model metadata.
-
-        :return: Model metadata.
-        :rtype: dict
-        """
-        project_meta_json = self.project_meta.to_json()
-        model_meta = {
-            "classes": [
-                item for item in project_meta_json["classes"] if item["title"] in self.classes
-            ]
-        }
-        return ProjectMeta.from_json(model_meta)
-
     @property
     def device(self) -> str:
         """
@@ -505,8 +491,7 @@ class TrainApp:
         self._download_project()
         # Step 3. Split Project
         self._split_project()
-        # Step 4. Convert Supervisely to X format
-        # Step 5. Download Model files
+        # Step 4. Download Model files
         self._download_model()
 
     def _finalize(self, experiment_info: dict) -> None:
@@ -528,13 +513,14 @@ class TrainApp:
         experiment_info = self._preprocess_artifacts(experiment_info)
 
         # Step3. Postprocess splits
-        splits_data = self._postprocess_splits()
+        train_splits_data = self._postprocess_splits()
 
         # Step 3. Upload artifacts
         self._set_text_status("uploading")
         remote_dir, file_info = self._upload_artifacts()
 
         # Step 4. Run Model Benchmark
+        model_meta = self.create_model_meta(experiment_info["task_type"])
         mb_eval_lnk_file_info, mb_eval_report, mb_eval_report_id, eval_metrics = (
             None,
             None,
@@ -543,6 +529,15 @@ class TrainApp:
         )
         if self.is_model_benchmark_enabled:
             try:
+                # Convert GT project
+                if self._app_options.get("auto_convert_classes", True):
+                    self._set_text_status("convert_gt_project")
+                    gt_project_id, bm_splits_data = self._convert_and_split_gt_project(
+                        experiment_info["task_type"]
+                    )
+                else:
+                    gt_project_id, bm_splits_data = None, train_splits_data
+
                 self._set_text_status("benchmark")
                 (
                     mb_eval_lnk_file_info,
@@ -550,7 +545,12 @@ class TrainApp:
                     mb_eval_report_id,
                     eval_metrics,
                 ) = self._run_model_benchmark(
-                    self.output_dir, remote_dir, experiment_info, splits_data
+                    self.output_dir,
+                    remote_dir,
+                    experiment_info,
+                    bm_splits_data,
+                    model_meta,
+                    gt_project_id,
                 )
             except Exception as e:
                 logger.error(f"Model benchmark failed: {e}")
@@ -571,8 +571,8 @@ class TrainApp:
         )
         self._generate_app_state(remote_dir, experiment_info)
         self._generate_hyperparameters(remote_dir, experiment_info)
-        self._generate_train_val_splits(remote_dir, splits_data)
-        self._generate_model_meta(remote_dir, experiment_info)
+        self._generate_train_val_splits(remote_dir, train_splits_data)
+        self._generate_model_meta(remote_dir, model_meta)
         self._upload_demo_files(remote_dir)
 
         # Step 7. Set output widgets
@@ -1200,9 +1200,14 @@ class TrainApp:
         logger.debug("Validation successful")
         return True, None
 
-    def _postprocess_splits(self) -> dict:
+    def _postprocess_splits(self, project_id: Optional[int] = None) -> dict:
         """
         Processes the train and val splits to generate the necessary data for the experiment_info.json file.
+
+        :param project_id: ID of the ground truth project for model benchmark. Provide only when cv task convertion is required.
+        :type project_id: Optional[int]
+        :return: Splits data.
+        :rtype: dict
         """
         val_dataset_ids = None
         val_images_ids = None
@@ -1212,10 +1217,30 @@ class TrainApp:
         split_method = self.gui.train_val_splits_selector.get_split_method()
         train_set, val_set = self._train_split, self._val_split
         if split_method == "Based on datasets":
-            val_dataset_ids = self.gui.train_val_splits_selector.get_val_dataset_ids()
-            train_dataset_ids = self.gui.train_val_splits_selector.get_train_dataset_ids
+            if project_id is None:
+                val_dataset_ids = self.gui.train_val_splits_selector.get_val_dataset_ids()
+                train_dataset_ids = self.gui.train_val_splits_selector.get_train_dataset_ids()
+            else:
+                src_datasets_map = {
+                    dataset.id: dataset
+                    for _, dataset in self._api.dataset.tree(self.project_info.id)
+                }
+                val_dataset_ids = self.gui.train_val_splits_selector.get_val_dataset_ids()
+                train_dataset_ids = self.gui.train_val_splits_selector.get_train_dataset_ids()
+
+                train_dataset_names = [src_datasets_map[ds_id].name for ds_id in train_dataset_ids]
+                val_dataset_names = [src_datasets_map[ds_id].name for ds_id in val_dataset_ids]
+
+                gt_datasets_map = {
+                    dataset.name: dataset.id for _, dataset in self._api.dataset.tree(project_id)
+                }
+                train_dataset_ids = [gt_datasets_map[ds_name] for ds_name in train_dataset_names]
+                val_dataset_ids = [gt_datasets_map[ds_name] for ds_name in val_dataset_names]
         else:
-            dataset_infos = [dataset for _, dataset in self._api.dataset.tree(self.project_id)]
+            if project_id is None:
+                project_id = self.project_id
+
+            dataset_infos = [dataset for _, dataset in self._api.dataset.tree(project_id)]
             ds_infos_dict = {}
             for dataset in dataset_infos:
                 if dataset.parent_id is not None:
@@ -1232,18 +1257,19 @@ class TrainApp:
                 image_infos = []
                 for dataset_name, image_names in image_names_per_dataset.items():
                     ds_info = ds_infos_dict[dataset_name]
-                    image_infos.extend(
-                        self._api.image.get_list(
-                            ds_info.id,
-                            filters=[
-                                {
-                                    "field": "name",
-                                    "operator": "in",
-                                    "value": image_names,
-                                }
-                            ],
+                    for names_batch in batched(image_names, 200):
+                        image_infos.extend(
+                            self._api.image.get_list(
+                                ds_info.id,
+                                filters=[
+                                    {
+                                        "field": "name",
+                                        "operator": "in",
+                                        "value": names_batch,
+                                    }
+                                ],
+                            )
                         )
-                    )
                 return image_infos
 
             val_image_infos = get_image_infos_by_split(ds_infos_dict, val_set)
@@ -1373,7 +1399,7 @@ class TrainApp:
             f"Uploading '{self._train_val_split_file}' to Team Files",
         )
 
-    def _generate_model_meta(self, remote_dir: str, experiment_info: dict) -> None:
+    def _generate_model_meta(self, remote_dir: str, model_meta: ProjectMeta) -> None:
         """
         Generates and uploads the model_meta.json file to the output directory.
 
@@ -1382,17 +1408,31 @@ class TrainApp:
         :param experiment_info: Information about the experiment results.
         :type experiment_info: dict
         """
-        # @TODO: Handle tags for classification tasks
         local_path = join(self.output_dir, self._model_meta_file)
         remote_path = join(remote_dir, self._model_meta_file)
 
-        sly_json.dump_json_file(self.model_meta.to_json(), local_path)
+        sly_json.dump_json_file(model_meta.to_json(), local_path)
         self._upload_file_to_team_files(
             local_path,
             remote_path,
             f"Uploading '{self._model_meta_file}' to Team Files",
         )
 
+    def create_model_meta(self, task_type: str):
+        """
+        Convert project meta according to task type.
+        """
+        names_to_delete = [
+            c.name for c in self.project_meta.obj_classes if c.name not in self.classes
+        ]
+        model_meta = self.project_meta.delete_obj_classes(names_to_delete)
+
+        if task_type == TaskType.OBJECT_DETECTION:
+            model_meta, _ = model_meta.to_detection_task(True)
+        elif task_type in [TaskType.INSTANCE_SEGMENTATION, TaskType.SEMANTIC_SEGMENTATION]:
+            model_meta, _ = model_meta.to_segmentation_task()  # @TODO: check background class
+        return model_meta
+
     def _generate_experiment_info(
         self,
         remote_dir: str,
@@ -1740,6 +1780,8 @@ class TrainApp:
         remote_artifacts_dir: str,
         experiment_info: dict,
         splits_data: dict,
+        model_meta: ProjectInfo,
+        gt_project_id: int = None,
     ) -> tuple:
         """
         Runs the Model Benchmark evaluation process. Model benchmark runs only in production mode.
@@ -1752,6 +1794,10 @@ class TrainApp:
         :type experiment_info: dict
         :param splits_data: Information about the train and val splits.
         :type splits_data: dict
+        :param model_meta: Model meta with object classes.
+        :type model_meta: ProjectInfo
+        :param gt_project_id: Ground truth project ID with converted shapes.
+        :type gt_project_id: int
         :return: Evaluation report, report ID and evaluation metrics.
         :rtype: tuple
         """
@@ -1767,6 +1813,7 @@ class TrainApp:
         supported_task_types = [
             TaskType.OBJECT_DETECTION,
             TaskType.INSTANCE_SEGMENTATION,
+            TaskType.SEMANTIC_SEGMENTATION,
         ]
         task_type = experiment_info["task_type"]
         if task_type not in supported_task_types:
@@ -1807,7 +1854,7 @@ class TrainApp:
                 "artifacts_dir": remote_artifacts_dir,
                 "model_name": experiment_info["model_name"],
                 "framework_name": self.framework_name,
-                "model_meta": self.model_meta.to_json(),
+                "model_meta": model_meta.to_json(),
             }
 
             logger.info(f"Deploy parameters: {self._benchmark_params}")
@@ -1827,12 +1874,15 @@ class TrainApp:
             train_images_ids = splits_data["train"]["images_ids"]
 
             bm = None
+            if gt_project_id is None:
+                gt_project_id = self.project_info.id
+
             if task_type == TaskType.OBJECT_DETECTION:
                 eval_params = ObjectDetectionEvaluator.load_yaml_evaluation_params()
                 eval_params = yaml.safe_load(eval_params)
                 bm = ObjectDetectionBenchmark(
                     self._api,
-                    self.project_info.id,
+                    gt_project_id,
                     output_dir=benchmark_dir,
                     gt_dataset_ids=benchmark_dataset_ids,
                     gt_images_ids=benchmark_images_ids,
@@ -1846,7 +1896,7 @@ class TrainApp:
                 eval_params = yaml.safe_load(eval_params)
                 bm = InstanceSegmentationBenchmark(
                     self._api,
-                    self.project_info.id,
+                    gt_project_id,
                     output_dir=benchmark_dir,
                     gt_dataset_ids=benchmark_dataset_ids,
                     gt_images_ids=benchmark_images_ids,
@@ -1860,7 +1910,7 @@ class TrainApp:
                 eval_params = yaml.safe_load(eval_params)
                 bm = SemanticSegmentationBenchmark(
                     self._api,
-                    self.project_info.id,
+                    gt_project_id,
                     output_dir=benchmark_dir,
                     gt_dataset_ids=benchmark_dataset_ids,
                     gt_images_ids=benchmark_images_ids,
@@ -2280,6 +2330,7 @@ class TrainApp:
             "metadata",
             "export_onnx",
             "export_trt",
+            "convert_gt_project",
         ],
     ):
 
@@ -2313,6 +2364,8 @@ class TrainApp:
             self.gui.training_process.validator_text.set("Validating experiment...", "info")
         elif status == "metadata":
             self.gui.training_process.validator_text.set("Generating training metadata...", "info")
+        elif status == "convert_gt_project":
+            self.gui.training_process.validator_text.set("Converting GT project...", "info")
 
     def _set_ws_progress_status(
         self,
@@ -2391,3 +2444,54 @@ class TrainApp:
             for runtime, path in export_weights.items()
         }
         return remote_export_weights
+
+    def _convert_and_split_gt_project(self, task_type: str):
+        # 1. Convert GT project to cv task
+        Project.download(
+            self._api, self.project_info.id, "tmp_project", save_images=False, save_image_info=True
+        )
+        project = Project("tmp_project", OpenMode.READ)
+
+        pr_prefix = ""
+        if task_type == TaskType.OBJECT_DETECTION:
+            Project.to_detection_task(project.directory, inplace=True)
+            pr_prefix = "[detection]: "
+        # @TODO: dont convert segmentation?
+        elif (
+            task_type == TaskType.INSTANCE_SEGMENTATION
+            or task_type == TaskType.SEMANTIC_SEGMENTATION
+        ):
+            Project.to_segmentation_task(project.directory, inplace=True)
+            pr_prefix = "[segmentation]: "
+
+        gt_project_info = self._api.project.create(
+            self.workspace_id,
+            f"{pr_prefix}{self.project_info.name}",
+            description=(
+                f"Converted ground truth project for trainig session: '{self.task_id}'. "
+                f"Original project id: '{self.project_info.id}. "
+                "Removing this project will affect model benchmark evaluation report."
+            ),
+            change_name_if_conflict=True,
+        )
+
+        # 3. Upload gt project to benchmark workspace
+        project = Project("tmp_project", OpenMode.READ)
+        self._api.project.update_meta(gt_project_info.id, project.meta)
+        for dataset in project.datasets:
+            dataset: Dataset
+            ds_info = self._api.dataset.create(
+                gt_project_info.id, dataset.name, change_name_if_conflict=True
+            )
+            for batch_names in batched(dataset.get_items_names(), 100):
+                img_infos = [dataset.get_item_info(name) for name in batch_names]
+                img_ids = [img_info.id for img_info in img_infos]
+                anns = [dataset.get_ann(name, project.meta) for name in batch_names]
+
+                img_infos = self._api.image.copy_batch(ds_info.id, img_ids)
+                img_ids = [img_info.id for img_info in img_infos]
+                self._api.annotation.upload_anns(img_ids, anns)
+
+        # 4. Match splits with original project
+        gt_split_data = self._postprocess_splits(gt_project_info.id)
+        return gt_project_info.id, gt_split_data

{supervisely-6.73.278 → supervisely-6.73.279/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.278
+Version: 6.73.279
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely