supervisely 6.73.285__tar.gz → 6.73.287__tar.gz

{supervisely-6.73.285/supervisely.egg-info → supervisely-6.73.287}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.285
+Version: 6.73.287
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/__init__.py

@@ -311,4 +311,4 @@ except Exception as e:
 # If new changes in Supervisely Python SDK require upgrade of the Supervisely instance
 # set a new value for the environment variable MINIMUM_INSTANCE_VERSION_FOR_SDK, otherwise
 # users can face compatibility issues, if the instance version is lower than the SDK version.
-os.environ["MINIMUM_INSTANCE_VERSION_FOR_SDK"] = "6.12.17"
+os.environ["MINIMUM_INSTANCE_VERSION_FOR_SDK"] = "6.12.23"

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/app/fastapi/subapp.py

@@ -926,19 +926,20 @@ class Application(metaclass=Singleton):
         self.test_client = TestClient(self._fastapi)
 
         if not headless:
-            templates = Jinja2Templates()
-            self.hot_reload = arel.HotReload([])
-            self._fastapi.add_websocket_route(
-                "/hot-reload", route=self.hot_reload, name="hot-reload"
-            )
-            self._fastapi.add_event_handler("startup", self.hot_reload.startup)
-            self._fastapi.add_event_handler("shutdown", self.hot_reload.shutdown)
+            if is_development() and hot_reload:
+                templates = Jinja2Templates()
+                self.hot_reload = arel.HotReload([])
+                self._fastapi.add_websocket_route(
+                    "/hot-reload", route=self.hot_reload, name="hot-reload"
+                )
+                self._fastapi.add_event_handler("startup", self.hot_reload.startup)
+                self._fastapi.add_event_handler("shutdown", self.hot_reload.shutdown)
 
-            # Setting HOTRELOAD=1 in template context, otherwise the HTML would not have the hot reload script.
-            templates.env.globals["HOTRELOAD"] = "1"
-            templates.env.globals["hot_reload"] = self.hot_reload
+                # Setting HOTRELOAD=1 in template context, otherwise the HTML would not have the hot reload script.
+                templates.env.globals["HOTRELOAD"] = "1"
+                templates.env.globals["hot_reload"] = self.hot_reload
 
-            logger.debug("Hot reload is enabled, use app.reload_page() to reload page.")
+                logger.debug("Hot reload is enabled, use app.reload_page() to reload page.")
 
             if is_production():
                 # to save offline session

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/base_benchmark.py

@@ -620,6 +620,9 @@ class BaseBenchmark:
     def get_eval_result(self):
         if self._eval_results is None:
             self._eval_results = self.evaluator.get_eval_result()
+            if not self._eval_results.inference_info:
+                self._eval_results.inference_info["gt_project_id"] = self.gt_project_info.id
+                self._eval_results.inference_info["dt_project_id"] = self.dt_project_info.id
         return self._eval_results
 
     def get_diff_project_info(self):

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/base_evaluator.py

@@ -16,7 +16,7 @@ class BaseEvalResult:
 
     def __init__(self, directory: Optional[str] = None):
         self.directory = directory
-        self.inference_info: Dict = None
+        self.inference_info: Dict = {}
         self.speedtest_info: Dict = None
         self.eval_data: Dict = None
         self.mp = None

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/instance_segmentation/text_templates.py

@@ -14,7 +14,7 @@ definitions = SimpleNamespace(
 )
 
 docs_url = (
-    "https://docs.supervisely.com/neural-networks/model-evaluation-benchmark/instant-segmentation"
+    "https://docs.supervisely.com/neural-networks/model-evaluation-benchmark/instance-segmentation"
 )
 
 # <i class="zmdi zmdi-check-circle" style="color: #13ce66; margin-right: 5px"></i>
@@ -44,6 +44,7 @@ markdown_overview = """
 {}
 - **IoU threshold**: {}
 - **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
+- **Averaging across IoU thresholds:** {}, <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
 """
@@ -60,7 +61,7 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
 """
 
-markdown_AP_custom_description = """> * AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
+markdown_AP_custom_description = """> *AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
 
 markdown_iou_per_class = """### IoU Threshold per Class
 

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/metric_provider.py

@@ -90,13 +90,16 @@ class MetricProvider:
         self.iouThrs = params["iouThrs"]
         self.recThrs = params["recThrs"]
 
+        # Evaluation params
         eval_params = params.get("evaluation_params", {})
         self.iou_threshold = eval_params.get("iou_threshold", 0.5)
         self.iou_threshold_idx = np.where(np.isclose(self.iouThrs, self.iou_threshold))[0][0]
-
-        # IoU per class (optional)
         self.iou_threshold_per_class = eval_params.get("iou_threshold_per_class")
         self.iou_idx_per_class = params.get("iou_idx_per_class")  # {cat id: iou_idx}
+        if self.iou_threshold_per_class is not None:
+            # TODO: temporary solution
+            eval_params["average_across_iou_thresholds"] = False
+        self.average_across_iou_thresholds = eval_params.get("average_across_iou_thresholds", True)
 
     def calculate(self):
         self.m_full = _MetricProvider(
@@ -199,9 +202,10 @@ class MetricProvider:
         }
 
     def AP_per_class(self):
-        s = self.coco_precision[:, :, :, 0, 2]
+        s = self.coco_precision[:, :, :, 0, 2].copy()
         s[s == -1] = np.nan
         ap = np.nanmean(s, axis=(0, 1))
+        ap = np.nan_to_num(ap, nan=0)
         return ap
 
     def AP_custom_per_class(self):
@@ -212,6 +216,7 @@ class MetricProvider:
                 s[:, cat_id - 1] = self.coco_precision[iou_idx, :, cat_id - 1, 0, 2]
         s[s == -1] = np.nan
         ap = np.nanmean(s, axis=0)
+        ap = np.nan_to_num(ap, nan=0)
         return ap
 
     def AP_custom(self):
@@ -284,6 +289,14 @@ class _MetricProvider:
         self.fp_not_confused_matches = [m for m in self.fp_matches if not m["miss_cls"]]
         self.ious = np.array([m["iou"] for m in self.tp_matches])
 
+        # Evaluation params
+        self.params = params
+        self.iou_idx_per_class = np.array(
+            [params["iou_idx_per_class"][cat_id] for cat_id in self.cat_ids]
+        )[:, None]
+        eval_params = params.get("evaluation_params", {})
+        self.average_across_iou_thresholds = eval_params.get("average_across_iou_thresholds", True)
+        
     def _init_counts(self):
         cat_ids = self.cat_ids
         iouThrs = self.iouThrs
@@ -328,14 +341,22 @@ class _MetricProvider:
         self.true_positives = true_positives
         self.false_negatives = false_negatives
         self.false_positives = false_positives
-        self.TP_count = int(self.true_positives[:, 0].sum(0))
-        self.FP_count = int(self.false_positives[:, 0].sum(0))
-        self.FN_count = int(self.false_negatives[:, 0].sum(0))
+        self.TP_count = int(self._take_iou_thresholds(true_positives).sum())
+        self.FP_count = int(self._take_iou_thresholds(false_positives).sum())
+        self.FN_count = int(self._take_iou_thresholds(false_negatives).sum())
 
+    def _take_iou_thresholds(self, x):
+        return np.take_along_axis(x, self.iou_idx_per_class, axis=1)
+    
     def base_metrics(self):
-        tp = self.true_positives
-        fp = self.false_positives
-        fn = self.false_negatives
+        if self.average_across_iou_thresholds:
+            tp = self.true_positives
+            fp = self.false_positives
+            fn = self.false_negatives
+        else:
+            tp = self._take_iou_thresholds(self.true_positives)
+            fp = self._take_iou_thresholds(self.false_positives)
+            fn = self._take_iou_thresholds(self.false_negatives)
         confuse_count = len(self.confused_matches)
 
         mAP = self.coco_mAP
@@ -358,9 +379,14 @@ class _MetricProvider:
         }
 
     def per_class_metrics(self):
-        tp = self.true_positives.mean(1)
-        fp = self.false_positives.mean(1)
-        fn = self.false_negatives.mean(1)
+        if self.average_across_iou_thresholds:
+            tp = self.true_positives.mean(1)
+            fp = self.false_positives.mean(1)
+            fn = self.false_negatives.mean(1)
+        else:
+            tp = self._take_iou_thresholds(self.true_positives).flatten()
+            fp = self._take_iou_thresholds(self.false_positives).flatten()
+            fn = self._take_iou_thresholds(self.false_negatives).flatten()
         pr = tp / (tp + fp)
         rc = tp / (tp + fn)
         f1 = 2 * pr * rc / (pr + rc)

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/text_templates.py

@@ -44,6 +44,7 @@ markdown_overview = """
 {}
 - **IoU threshold**: {}
 - **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
+- **Averaging across IoU thresholds:** {}, <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
 """
@@ -65,7 +66,7 @@ Here, we comprehensively assess the model's performance by presenting a broad se
 - **Calibration Score**: This score represents the consistency of predicted probabilities (or <abbr title="{}">confidence scores</abbr>) made by the model. We evaluate how well predicted probabilities align with actual outcomes. A well-calibrated model means that when it predicts an object with, say, 80% confidence, approximately 80% of those predictions should actually be correct.
 """
 
-markdown_AP_custom_description = """> * AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
+markdown_AP_custom_description = """> *AP_custom - Average Precision with different IoU thresholds for each class, that was set in evaluation params by the user."""
 
 markdown_iou_per_class = """### IoU Threshold per Class
 

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/vis_metrics/confidence_score.py

@@ -72,13 +72,13 @@ class ConfidenceScore(DetectionVisMetric):
             self.eval_result.dfsp_down,
             x="scores",
             y=["precision", "recall", "f1"],
-            labels={"value": "Value", "variable": "Metric", "scores": "Confidence Score"},
+            labels={"value": "Metric", "variable": "Metric", "scores": "Confidence Score"},
             width=None,
             height=500,
             color_discrete_map=color_map,
         )
         fig.update_traces(
-            hovertemplate="Confidence Score: %{x:.2f}<br>Value: %{y:.2f}<extra></extra>"
+            hovertemplate="Confidence score: %{x:.2f}<br>Metric: %{y:.2f}<extra></extra>"
         )
         fig.update_layout(yaxis=dict(range=[0, 1]), xaxis=dict(range=[0, 1], tick0=0, dtick=0.1))
 

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/vis_metrics/f1_score_at_different_iou.py

@@ -39,7 +39,7 @@ class F1ScoreAtDifferentIOU(DetectionVisMetric):
             np.concatenate([self.eval_result.dfsp_down["scores"].values[:, None], f1s_down.T], 1),
             columns=["scores"] + iou_names,
         )
-        labels = {"value": "Value", "variable": "IoU threshold", "scores": "Confidence Score"}
+        labels = {"value": "F1-score", "variable": "IoU threshold", "scores": "Confidence Score"}
 
         fig = px.line(
             df,
@@ -51,19 +51,19 @@ class F1ScoreAtDifferentIOU(DetectionVisMetric):
             height=500,
         )
         fig.update_traces(
-            hovertemplate="Confidence Score: %{x:.2f}<br>Value: %{y:.2f}<extra></extra>"
+            hovertemplate="Confidence Score: %{x:.2f}<br>F1-score: %{y:.2f}<extra></extra>"
         )
         fig.update_layout(yaxis=dict(range=[0, 1]), xaxis=dict(range=[0, 1], tick0=0, dtick=0.1))
 
         # add annotations for maximum F1-Score for each IoU threshold
         for i, iou in enumerate(iou_names):
-            argmax_f1 = f1s[i].argmax()
+            argmax_f1 = np.nanargmax(f1s[i])
             max_f1 = f1s[i][argmax_f1]
             score = self.eval_result.mp.m_full.score_profile["scores"][argmax_f1]
             fig.add_annotation(
                 x=score,
                 y=max_f1,
-                text=f"Best score: {score:.2f}",
+                text=f"Best conf: {score:.2f}",
                 showarrow=True,
                 arrowhead=1,
                 arrowcolor="black",

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/vis_metrics/outcome_counts_per_class.py

@@ -86,11 +86,10 @@ class PerClassOutcomeCounts(DetectionVisMetric):
         import plotly.express as px  # pylint: disable=import-error
 
         # Per-class Counts
-        iou_thres = 0
-
-        tp = self.eval_result.mp.true_positives[:, iou_thres]
-        fp = self.eval_result.mp.false_positives[:, iou_thres]
-        fn = self.eval_result.mp.false_negatives[:, iou_thres]
+        mp = self.eval_result.mp
+        tp = mp.m._take_iou_thresholds(mp.true_positives).flatten()
+        fp = mp.m._take_iou_thresholds(mp.false_positives).flatten()
+        fn = mp.m._take_iou_thresholds(mp.false_negatives).flatten()
 
         # normalize
         support = tp + fn

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/vis_metrics/overview.py

@@ -20,7 +20,7 @@ class Overview(DetectionVisMetric):
         url = self.eval_result.inference_info.get("checkpoint_url")
         link_text = self.eval_result.inference_info.get("custom_checkpoint_path")
         if link_text is None:
-            link_text = url
+            link_text = url or ""
         link_text = link_text.replace("_", "\_")
 
         model_name = self.eval_result.inference_info.get("model_name") or "Custom"
@@ -31,6 +31,7 @@ class Overview(DetectionVisMetric):
 
         # link to scroll to the optimal confidence section
         opt_conf_url = self.vis_texts.docs_url + "#f1-optimal-confidence-threshold"
+        average_url = self.vis_texts.docs_url + "#averaging-iou-thresholds"
 
         iou_threshold = self.eval_result.mp.iou_threshold
         if self.eval_result.different_iou_thresholds_per_class:
@@ -52,6 +53,8 @@ class Overview(DetectionVisMetric):
             iou_threshold,
             round(self.eval_result.mp.f1_optimal_conf, 4),
             opt_conf_url,
+            self.eval_result.mp.average_across_iou_thresholds,
+            average_url,
             self.vis_texts.docs_url,
         ]
 

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/benchmark/object_detection/vis_metrics/precision_avg_per_class.py

@@ -32,8 +32,7 @@ class PerClassAvgPrecision(DetectionVisMetric):
         import plotly.express as px  # pylint: disable=import-error
 
         # AP per-class
-        ap_per_class = self.eval_result.mp.coco_precision[:, :, :, 0, 2].mean(axis=(0, 1))
-        ap_per_class[ap_per_class == -1] = 0  # -1 is a placeholder for no GT
+        ap_per_class = self.eval_result.mp.AP_per_class()
         labels = dict(r="Average Precision", theta="Class")
         fig = px.scatter_polar(
             r=ap_per_class,

{supervisely-6.73.285 → supervisely-6.73.287}/supervisely/nn/inference/inference.py

@@ -2654,6 +2654,7 @@ class Inference:
 
             # Ger rid of `pending_results` to less response size
             inference_request["pending_results"] = []
+            inference_request.pop("lock", None)
             return inference_request
 
         @server.post(f"/pop_inference_results")

{supervisely-6.73.285 → supervisely-6.73.287/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.285
+Version: 6.73.287
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely