supervisely 6.73.200__tar.gz → 6.73.202__tar.gz

{supervisely-6.73.200/supervisely.egg-info → supervisely-6.73.202}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.200
+Version: 6.73.202
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/api/file_api.py

@@ -806,6 +806,40 @@ class FileApi(ModuleApiBase):
             api.file.upload_bulk(8, src_paths, dst_remote_paths)
         """
 
+        def _group_files_generator(
+            src_paths: List[str], dst_paths: List[str], limit: int = 20 * 1024 * 1024
+        ):
+            if limit is None:
+                return src_paths, dst_paths
+            group_src = []
+            group_dst = []
+            total_size = 0
+            for src, dst in zip(src_paths, dst_paths):
+                size = os.path.getsize(src)
+                if total_size > 0 and total_size + size > limit:
+                    yield group_src, group_dst
+                    group_src = []
+                    group_dst = []
+                    total_size = 0
+                group_src.append(src)
+                group_dst.append(dst)
+                total_size += size
+            if total_size > 0:
+                yield group_src, group_dst
+
+        file_infos = []
+        for src, dst in _group_files_generator(src_paths, dst_paths):
+            file_infos.extend(self._upload_bulk(team_id, src, dst, progress_cb))
+        return file_infos
+
+    def _upload_bulk(
+        self,
+        team_id: int,
+        src_paths: List[str],
+        dst_paths: List[str],
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+    ) -> List[FileInfo]:
+
         def path_to_bytes_stream(path):
             return open(path, "rb")
 
@@ -1338,7 +1372,7 @@ class FileApi(ModuleApiBase):
         """
         if not remote_dir.startswith("/"):
             remote_dir = "/" + remote_dir
-            
+
         if self.dir_exists(team_id, remote_dir):
             if change_name_if_conflict is True:
                 res_remote_dir = self.get_free_dir_name(team_id, remote_dir)

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/base_benchmark.py

@@ -449,6 +449,28 @@ class BaseBenchmark:
         vis.visualize()
 
     def _get_or_create_diff_project(self) -> Tuple[ProjectInfo, bool]:
+
+        dt_ds_id_to_diff_ds_info = {}
+
+        def _get_or_create_diff_dataset(dt_dataset_id, dt_datasets):
+            if dt_dataset_id in dt_ds_id_to_diff_ds_info:
+                return dt_ds_id_to_diff_ds_info[dt_dataset_id]
+            dt_dataset = dt_datasets[dt_dataset_id]
+            if dt_dataset.parent_id is None:
+                diff_dataset = self.api.dataset.create(
+                    diff_project_info.id,
+                    dt_dataset.name,
+                )
+            else:
+                parent_dataset = _get_or_create_diff_dataset(dt_dataset.parent_id, dt_datasets)
+                diff_dataset = self.api.dataset.create(
+                    diff_project_info.id,
+                    dt_dataset.name,
+                    parent_id=parent_dataset.id,
+                )
+            dt_ds_id_to_diff_ds_info[dt_dataset_id] = diff_dataset
+            return diff_dataset
+
         diff_project_name = self._generate_diff_project_name(self.dt_project_info.name)
         diff_workspace_id = self.dt_project_info.workspace_id
         diff_project_info = self.api.project.get_info_by_name(
@@ -460,8 +482,12 @@ class BaseBenchmark:
             diff_project_info = self.api.project.create(
                 diff_workspace_id, diff_project_name, change_name_if_conflict=True
             )
-            for dataset in self.api.dataset.get_list(self.dt_project_info.id):
-                self.api.dataset.create(diff_project_info.id, dataset.name)
+            dt_datasets = {
+                ds.id: ds
+                for ds in self.api.dataset.get_list(self.dt_project_info.id, recursive=True)
+            }
+            for dataset in dt_datasets:
+                _get_or_create_diff_dataset(dataset, dt_datasets)
         return diff_project_info, is_existed
 
     def upload_visualizations(self, dest_dir: str):

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/coco_utils/sly2coco.py

@@ -7,7 +7,7 @@ import numpy as np
 
 from supervisely import Bitmap
 from supervisely._utils import batched
-from supervisely.project.project import Project, OpenMode
+from supervisely.project.project import Dataset, OpenMode, Project
 
 
 def sly2coco(
@@ -20,11 +20,7 @@ def sly2coco(
 ):
     from pycocotools import mask as maskUtils  # pylint: disable=import-error
 
-    datasets = [
-        name
-        for name in os.listdir(sly_project_path)
-        if os.path.isdir(pjoin(sly_project_path, name))
-    ]
+    project = Project(sly_project_path, mode=OpenMode.READ)
 
     # Categories
     meta_path = pjoin(sly_project_path, "meta.json")
@@ -52,13 +48,13 @@ def sly2coco(
     images = []
     annotations = []
     annotation_id = 1
-    project = Project(sly_project_path, mode=OpenMode.READ)
     total = project.total_items
     with progress(message="Evaluation: Converting to COCO format", total=total) as pbar:
         img_id = 1
-        for dataset_name in datasets:
-            ann_path = pjoin(sly_project_path, dataset_name, "ann")
-            imginfo_path = pjoin(sly_project_path, dataset_name, "img_info")
+        for dataset in sorted(project.datasets, key=lambda x: x.name):
+            dataset: Dataset
+            ann_path = dataset.ann_dir
+            imginfo_path = dataset.img_info_dir
             ann_files = sorted(os.listdir(ann_path))
             for batch in batched(ann_files, 30):
                 for ann_file in batch:
@@ -75,7 +71,7 @@ def sly2coco(
                         "width": img_w,
                         "height": img_h,
                         "sly_id": img_info["id"],
-                        "dataset": dataset_name,
+                        "dataset": dataset.name,
                     }
                     images.append(img)
                     for label in ann["objects"]:

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/evaluation/instance_segmentation_evaluator.py

@@ -1,22 +1,28 @@
 import os
+
 from supervisely.io.json import dump_json_file
-from supervisely.nn.benchmark.evaluation import BaseEvaluator
 from supervisely.nn.benchmark.coco_utils import read_coco_datasets, sly2coco
+from supervisely.nn.benchmark.evaluation import BaseEvaluator
 from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
 
 
 class InstanceSegmentationEvaluator(BaseEvaluator):
     def evaluate(self):
-        self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
+        try:
+            self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
+        except AssertionError as e:
+            raise ValueError(
+                f"{e}. Please make sure that your GT and DT projects are correct. "
+                "If GT project has nested datasets and DT project was crated with NN app, "
+                "try to use newer version of NN app."
+            )
+
         self._dump_datasets()
         self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
         with self.pbar(message="Evaluation: Calculating metrics", total=10) as p:
             self.eval_data = calculate_metrics(
-                self.cocoGt,
-                self.cocoDt,
-                iouType="segm",
-                progress_cb=p.update
-                )
+                self.cocoGt, self.cocoDt, iouType="segm", progress_cb=p.update
+            )
         self._dump_eval_results()
 
     def _convert_to_coco(self):
@@ -41,8 +47,12 @@ class InstanceSegmentationEvaluator(BaseEvaluator):
                 "Not found any predictions. "
                 "Please make sure that your model produces predictions."
             )
-        assert cocoDt_json['categories'] == cocoGt_json['categories']
-        assert [x['id'] for x in cocoDt_json['images']] == [x['id'] for x in cocoGt_json['images']]
+        assert (
+            cocoDt_json["categories"] == cocoGt_json["categories"]
+        ), "Object classes in GT and DT projects are different"
+        assert [f'{x["dataset"]}/{x["file_name"]}' for x in cocoDt_json["images"]] == [
+            f'{x["dataset"]}/{x["file_name"]}' for x in cocoGt_json["images"]
+        ], "Images in GT and DT projects are different"
         return cocoGt_json, cocoDt_json
 
     def _dump_datasets(self):

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/evaluation/object_detection_evaluator.py

@@ -8,7 +8,14 @@ from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
 
 class ObjectDetectionEvaluator(BaseEvaluator):
     def evaluate(self):
-        self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
+        try:
+            self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
+        except AssertionError as e:
+            raise ValueError(
+                f"{e}. Please make sure that your GT and DT projects are correct. "
+                "If GT project has nested datasets and DT project was crated with NN app, "
+                "try to use newer version of NN app."
+            )
         self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
         with self.pbar(message="Evaluation: Calculating metrics", total=10) as p:
             self.eval_data = calculate_metrics(
@@ -42,7 +49,9 @@ class ObjectDetectionEvaluator(BaseEvaluator):
         assert (
             cocoDt_json["categories"] == cocoGt_json["categories"]
         ), "Classes in GT and Pred projects must be the same"
-        assert [x["id"] for x in cocoDt_json["images"]] == [x["id"] for x in cocoGt_json["images"]]
+        assert [f'{x["dataset"]}/{x["file_name"]}' for x in cocoDt_json["images"]] == [
+            f'{x["dataset"]}/{x["file_name"]}' for x in cocoGt_json["images"]
+        ], "Images in GT and DT projects are different"
         return cocoGt_json, cocoDt_json
 
     def _dump_eval_results(self):

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/text_templates/instance_segmentation_text.py

@@ -49,7 +49,7 @@ Learn more about Model Benchmark, implementation details, and how to use the cha
 
 markdown_key_metrics = """## Key Metrics
 
-Here, we comprehensively assess the model's performance by presenting a broad set of metrics, including mAP (mean Average Precision), Precision, Recall, IoU (Intersection over Union), Classification Accuracy, Calibration Score, and Inference Speed.
+Here, we comprehensively assess the model's performance by presenting a broad set of metrics, including mAP (mean Average Precision), Precision, Recall, IoU (Intersection over Union), Classification Accuracy and Calibration Score.
 
 - **Mean Average Precision (mAP)**: A comprehensive metric of detection and instance segmentation performance. mAP calculates the <abbr title="{}">average precision</abbr> across all classes at different levels of <abbr title="{}">IoU thresholds</abbr> and precision-recall trade-offs. In other words, it evaluates the performance of a model by considering its ability to detect and localize objects accurately across multiple IoU thresholds and object categories.
 - **Precision**: Precision indicates how often the model's predictions are actually correct when it predicts an object. This calculates the ratio of correct predictions to the total number of predictions made by the model.
@@ -57,7 +57,6 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Intersection over Union (IoU)**: IoU measures the overlap between two masks: one predicted by the model and one from the ground truth. It is calculated as the area of intersection between the predicted mask and the ground truth mask, divided by the area of their union. A higher IoU score indicates better alignment between the predicted and ground truth masks.
 - **Classification Accuracy**: We additionally measure the classification accuracy of an instance segmentation model. This metric represents the percentage of correctly labeled instances among all instances where the predicted segmentation masks accurately match the ground truth masks (with an IoU greater than 0.5, regardless of class).
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
-- **Inference Speed**: The number of frames per second (FPS) the model can process, measured with a batch size of 1. The inference speed is important in applications, where real-time inference is required. Additionally, slower models pour more GPU resources, so their inference cost is higher.
 """
 
 markdown_explorer = """## Explore Predictions

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/text_templates/object_detection_text.py

@@ -49,7 +49,7 @@ Learn more about Model Benchmark, implementation details, and how to use the cha
 
 markdown_key_metrics = """## Key Metrics
 
-Here, we comprehensively assess the model's performance by presenting a broad set of metrics, including mAP (mean Average Precision), Precision, Recall, IoU (Intersection over Union), Classification Accuracy, Calibration Score, and Inference Speed.
+Here, we comprehensively assess the model's performance by presenting a broad set of metrics, including mAP (mean Average Precision), Precision, Recall, IoU (Intersection over Union), Classification Accuracy and Calibration Score.
 
 - **Mean Average Precision (mAP)**: A comprehensive metric of detection performance. mAP calculates the <abbr title="{}">average precision</abbr> across all classes at different levels of <abbr title="{}">IoU thresholds</abbr> and precision-recall trade-offs. In other words, it evaluates the performance of a model by considering its ability to detect and localize objects accurately across multiple IoU thresholds and object categories.
 - **Precision**: Precision indicates how often the model's predictions are actually correct when it predicts an object. This calculates the ratio of correct detections to the total number of detections made by the model.
@@ -57,7 +57,6 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Intersection over Union (IoU)**: IoU measures how closely predicted bounding boxes match the actual (ground truth) bounding boxes. It is calculated as the area of overlap between the predicted bounding box and the ground truth bounding box, divided by the area of union of these bounding boxes.
 - **Classification Accuracy**: We additionally measure the classification accuracy of an object detection model. This metric represents the percentage of correctly labeled instances among all correctly localized bounding boxes (where the IoU for each box is greater than 0.5, regardless of class).
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
-- **Inference Speed**: The number of frames per second (FPS) the model can process, measured with a batch size of 1. The inference speed is important in applications, where real-time object detection is required. Additionally, slower models pour more GPU resources, so their inference cost is higher.
 """
 
 markdown_explorer = """## Explore Predictions

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/vis_metric_base.py

@@ -291,7 +291,7 @@ class MetricVis:
                 zip(image_infos, ann_infos, project_metas)
             ):
                 image_name = image_info.name
-                image_url = image_info.full_storage_url
+                image_url = image_info.preview_url
                 is_ignore = True if idx in [0, 1] else False
                 widget.gallery.append(
                     title=image_name,

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/vis_metrics/explorer_grid.py

@@ -37,7 +37,7 @@ class ExplorerGrid(MetricVis):
 
         for idx, (pred_image, ann_info) in enumerate(zip(pred_image_infos, ann_infos)):
             image_name = pred_image.name
-            image_url = pred_image.full_storage_url
+            image_url = pred_image.preview_url
             widget.gallery.append(
                 title=image_name,
                 image_url=image_url,

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/vis_metrics/model_predictions.py

@@ -33,7 +33,7 @@ class ModelPredictions(MetricVis):
         dt_project_id = self._loader.dt_project_info.id
 
         tmp = set()
-        for dt_dataset in self._loader._api.dataset.get_list(dt_project_id):
+        for dt_dataset in self._loader._api.dataset.get_list(dt_project_id, recursive=True):
             names = [x.name for x in self._loader._api.image.get_list(dt_dataset.id)]
             tmp.update(names)
         df = self._loader.mp.prediction_table().round(2)

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/vis_metrics/precision.py

@@ -42,6 +42,7 @@ class Precision(MetricVis):
             y="precision",
             # title="Per-class Precision (Sorted by F1)",
             color="precision",
+            range_color=[0, 1],
             color_continuous_scale="Plasma",
         )
         fig.update_traces(hovertemplate="Class: %{x}<br>Precision: %{y:.2f}<extra></extra>")

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/vis_metrics/recall.py

@@ -40,6 +40,7 @@ class Recall(MetricVis):
             y="recall",
             # title="Per-class Recall (Sorted by F1)",
             color="recall",
+            range_color=[0, 1],
             color_continuous_scale="Plasma",
         )
         fig.update_traces(hovertemplate="Class: %{x}<br>Recall: %{y:.2f}<extra></extra>")

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/benchmark/visualization/visualizer.py

@@ -361,8 +361,19 @@ class Visualizer:
 
         gt_project = Project(gt_project_path, OpenMode.READ)
         pred_project = Project(pred_project_path, OpenMode.READ)
+        diff_dataset_id_to_info = {
+            ds.id: ds
+            for ds in self._api.dataset.get_list(self.diff_project_info.id, recursive=True)
+        }
+
+        def _get_full_name(ds_id: int):
+            ds_info = diff_dataset_id_to_info[ds_id]
+            if ds_info.parent_id is None:
+                return ds_info.name
+            return f"{_get_full_name(ds_info.parent_id)}/{ds_info.name}"
+
         diff_dataset_name_to_info = {
-            ds.name: ds for ds in self._api.dataset.get_list(self.diff_project_info.id)
+            _get_full_name(ds_id): ds_info for ds_id, ds_info in diff_dataset_id_to_info.items()
         }
 
         matched_id_map = self._get_matched_id_map()  # dt_id -> gt_id
@@ -378,6 +389,13 @@ class Visualizer:
         with self.pbar(
             message="Visualizations: Creating diff_project", total=pred_project.total_items
         ) as progress:
+            logger.debug(
+                "Creating diff project data",
+                extra={
+                    "pred_project": [ds.name for ds in pred_project.datasets],
+                    "gt_project": [ds.name for ds in gt_project.datasets],
+                },
+            )
             for pred_dataset in pred_project.datasets:
                 pred_dataset: Dataset
                 gt_dataset: Dataset = gt_project.datasets.get(pred_dataset.name)
@@ -460,8 +478,19 @@ class Visualizer:
         gt_project_path, pred_project_path = self._benchmark._download_projects(save_images=False)
         gt_project = Project(gt_project_path, OpenMode.READ)
         pred_project = Project(pred_project_path, OpenMode.READ)
+        diff_dataset_id_to_info = {
+            ds.id: ds
+            for ds in self._api.dataset.get_list(self.diff_project_info.id, recursive=True)
+        }
+
+        def _get_full_name(ds_id: int):
+            ds_info = diff_dataset_id_to_info[ds_id]
+            if ds_info.parent_id is None:
+                return ds_info.name
+            return f"{_get_full_name(ds_info.parent_id)}/{ds_info.name}"
+
         diff_dataset_name_to_info = {
-            ds.name: ds for ds in self._api.dataset.get_list(self.diff_project_info.id)
+            _get_full_name(ds_id): ds_info for ds_id, ds_info in diff_dataset_id_to_info.items()
         }
 
         for pred_dataset in pred_project.datasets:

{supervisely-6.73.200 → supervisely-6.73.202}/supervisely/nn/inference/inference.py

@@ -1498,9 +1498,27 @@ class Inference:
             if src_dataset_id in new_dataset_id:
                 return new_dataset_id[src_dataset_id]
             dataset_info = api.dataset.get_info_by_id(src_dataset_id)
-            output_dataset_id = api.dataset.copy(
-                output_project_id, src_dataset_id, dataset_info.name, change_name_if_conflict=True
-            ).id
+            if dataset_info.parent_id is None:
+                output_dataset_id = api.dataset.copy(
+                    output_project_id,
+                    src_dataset_id,
+                    dataset_info.name,
+                    change_name_if_conflict=True,
+                ).id
+            else:
+                parent_dataset_id = _get_or_create_new_dataset(
+                    output_project_id, dataset_info.parent_id
+                )
+                output_dataset_info = api.dataset.create(
+                    output_project_id, dataset_info.name, parent_id=parent_dataset_id
+                )
+                api.image.copy_batch_optimized(
+                    dataset_info.id,
+                    images_infos_dict[dataset_info.id],
+                    output_dataset_info.id,
+                    with_annotations=False,
+                )
+                output_dataset_id = output_dataset_info.id
             new_dataset_id[src_dataset_id] = output_dataset_id
             return output_dataset_id
 
@@ -1775,7 +1793,9 @@ class Inference:
         stop = False
 
         def image_batch_generator(batch_size):
-            logger.debug(f"image_batch_generator. images_infos={len(images_infos)}, batch_size={batch_size}")
+            logger.debug(
+                f"image_batch_generator. images_infos={len(images_infos)}, batch_size={batch_size}"
+            )
             batch = []
             while True:
                 for image_info in images_infos:

{supervisely-6.73.200 → supervisely-6.73.202/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.200
+Version: 6.73.202
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely