supervisely 6.73.238__py3-none-any.whl → 6.73.240__py3-none-any.whl

supervisely/nn/benchmark/instance_segmentation/evaluator.py

@@ -0,0 +1,58 @@
+import os
+from pathlib import Path
+
+from supervisely.io.json import dump_json_file
+from supervisely.nn.benchmark.object_detection.evaluator import (
+    ObjectDetectionEvalResult,
+    ObjectDetectionEvaluator,
+)
+from supervisely.nn.benchmark.object_detection.metric_provider import MetricProvider
+from supervisely.nn.benchmark.utils import calculate_metrics, read_coco_datasets
+
+
+class InstanceSegmentationEvalResult(ObjectDetectionEvalResult):
+    mp_cls = MetricProvider
+
+
+class InstanceSegmentationEvaluator(ObjectDetectionEvaluator):
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"
+    eval_result_cls = InstanceSegmentationEvalResult
+    accepted_shapes = ["polygon", "bitmap"]
+
+    def evaluate(self):
+        try:
+            self.cocoGt_json, self.cocoDt_json = self._convert_to_coco()
+        except AssertionError as e:
+            raise ValueError(
+                f"{e}. Please make sure that your GT and DT projects are correct. "
+                "If GT project has nested datasets and DT project was crated with NN app, "
+                "try to use newer version of NN app."
+            )
+
+        self._dump_datasets()
+        self.cocoGt, self.cocoDt = read_coco_datasets(self.cocoGt_json, self.cocoDt_json)
+        with self.pbar(message="Evaluation: Calculating metrics", total=5) as p:
+            self.eval_data = calculate_metrics(
+                self.cocoGt,
+                self.cocoDt,
+                iouType="segm",
+                progress_cb=p.update,
+                evaluation_params=self.evaluation_params,
+            )
+        self._dump_eval_results()
+
+    def _dump_eval_results(self):
+        _, _, eval_data_path = self._get_eval_paths()
+        self._dump_pickle(self.eval_data, eval_data_path)
+
+    def _get_eval_paths(self):
+        base_dir = self.result_dir
+        cocoGt_path = os.path.join(base_dir, "cocoGt.json")
+        cocoDt_path = os.path.join(base_dir, "cocoDt.json")
+        eval_data_path = os.path.join(base_dir, "eval_data.pkl")
+        return cocoGt_path, cocoDt_path, eval_data_path
+
+    def _dump_datasets(self):
+        cocoGt_path, cocoDt_path, _ = self._get_eval_paths()
+        dump_json_file(self.cocoGt_json, cocoGt_path, indent=None)
+        dump_json_file(self.cocoDt_json, cocoDt_path, indent=None)

supervisely/nn/benchmark/{visualization/text_templates/instance_segmentation_text.py → instance_segmentation/text_templates.py}

@@ -13,6 +13,10 @@ definitions = SimpleNamespace(
     iou_threshold="The IoU threshold is a predefined value (set to 0.5 in many benchmarks) that determines the minimum acceptable IoU score for a predicted mask to be considered a correct prediction. When the IoU of a predicted mask and actual mask is higher than this IoU threshold, the prediction is considered correct. Some metrics will evaluate the model with different IoU thresholds to provide more insights about the model's performance.",
 )
 
+docs_url = (
+    "https://docs.supervisely.com/neural-networks/model-evaluation-benchmark/instant-segmentation"
+)
+
 # <i class="zmdi zmdi-check-circle" style="color: #13ce66; margin-right: 5px"></i>
 clickable_label = """
 > <span style="color: #5a6772">
@@ -35,17 +39,14 @@ markdown_overview = """
 - **Architecture**: {}
 - **Task type**: {}
 - **Runtime**: {}
-- **Checkpoint file**: <a href="{}" target="_blank">{}</a>
+- **Checkpoint file**: <a class="checkpoint-url" href="{}" target="_blank">{}</a>
 - **Ground Truth project**: <a href="/projects/{}/datasets" target="_blank">{}</a>, {}{}
 {}
+- **IoU threshold**: {}
+- **Optimal confidence threshold**: {} (calculated automatically), <a href="{}" target="_blank">learn more</a>.
 
 Learn more about Model Benchmark, implementation details, and how to use the charts in our <a href="{}" target="_blank">Technical Report</a>.
 """
-# - **Model**: {}
-# - **Training dataset (?)**: COCO 2017 train
-# - **Model classes (?)**: (80): a, b, c, … (collapse)
-# - **Model weights (?)**: [/path/to/yolov8l.pt]()
-# - **License (?)**: AGPL-3.0
 
 markdown_key_metrics = """## Key Metrics
 
@@ -67,17 +68,6 @@ In this section you can visually assess the model performance through examples.
 > Filtering options allow you to adjust the confidence threshold (only for predictions) and the model's false outcomes (only for differences). Differences are calculated only for the optimal confidence threshold, allowing you to focus on the most accurate predictions made by the model.
 """
 
-markdown_predictions_gallery = """
-
-"""
-# You can choose one of the sorting method:
-
-# - **Auto**: The algorithm is trying to gather a diverse set of images that illustrate the model's performance across various scenarios.
-# - **Least accurate**: Displays images where the model made more errors.
-# - **Most accurate**: Displays images where the model made fewer or no errors.
-# - **Dataset order**: Displays images in the original order of the dataset.
-# """
-
 markdown_predictions_table = """### Prediction details for every image
 
 The table helps you in finding samples with specific cases of interest. You can sort by parameters such as the number of predictions, or specific a metric, e.g, recall, then click on a row to view this image and predictions.
@@ -88,27 +78,15 @@ The table helps you in finding samples with specific cases of interest. You can
 > Click on the row to view the image with **Ground Truth**, **Prediction**, or the **Difference** annotations.
 """
 
-markdown_what_is = """
-"""
-
-markdown_experts = """
-"""
-
-markdown_how_to_use = """
-"""
-
-markdown_outcome_counts = (
-    """## Outcome Counts
+markdown_outcome_counts = """## Outcome Counts
 
 This chart is used to evaluate the overall model performance by breaking down all predictions into <abbr title="{}">True Positives</abbr> (TP), <abbr title="{}">False Positives</abbr> (FP), and <abbr title="{}">False Negatives</abbr> (FN). This helps to visually assess the type of errors the model often encounters.
 
 """
-    + clickable_label
-)
 
 markdown_R = """## Recall
 
-This section measures the ability of the model to find **all relevant instances in the dataset**. In other words, it answers the question: “Of all instances in the dataset, how many of them is the model managed to find out?”
+This section measures the ability of the model to find **all relevant instances in the dataset**. In other words, it answers the question: "Of all instances in the dataset, how many of them is the model managed to find out?"
 
 To measure this, we calculate **Recall**. Recall counts errors, when the model does not predict an object that actually is present in a dataset and should be predicted. Recall is calculated as the portion of correct predictions (true positives) over all instances in the dataset (true positives + false negatives).
 """
@@ -118,8 +96,7 @@ notification_recall = {
     "description": "The model correctly found <b>{} of {}</b> total instances in the dataset.",
 }
 
-markdown_R_perclass = (
-    """### Per-class Recall
+markdown_R_perclass = """### Per-class Recall
 
 This chart further analyzes Recall, breaking it down to each class in separate.
 
@@ -128,15 +105,13 @@ Since the overall recall is calculated as an average across all classes, we prov
 _Bars in the chart are sorted by <abbr title="{}">F1-score</abbr> to keep a unified order of classes between different charts._
 
 """
-    + clickable_label
-)
 
 
 markdown_P = """## Precision
 
-This section measures the accuracy of all predictions made by the model. In other words, it answers the question: “Of all predictions made by the model, how many of them are actually correct?”.
+This section measures the accuracy of all predictions made by the model. In other words, it answers the question: "Of all predictions made by the model, how many of them are actually correct?".
 
-To measure this, we calculate **Precision**. Precision counts errors, when the model predicts an object, but the image has no objects of the predicted class in this place. Precision is calculated as a portion of correct predictions (true positives) over all model’s predictions (true positives + false positives).
+To measure this, we calculate **Precision**. Precision counts errors, when the model predicts an object, but the image has no objects of the predicted class in this place. Precision is calculated as a portion of correct predictions (true positives) over all model's predictions (true positives + false positives).
 """
 
 notification_precision = {
@@ -144,8 +119,7 @@ notification_precision = {
     "description": "The model correctly predicted <b>{} of {}</b> predictions made by the model in total.",
 }
 
-markdown_P_perclass = (
-    """### Per-class Precision
+markdown_P_perclass = """### Per-class Precision
 
 This chart further analyzes Precision, breaking it down to each class in separate.
 
@@ -154,20 +128,15 @@ Since the overall precision is computed as an average across all classes, we pro
 _Bars in the chart are sorted by <abbr title="{}">F1-score</abbr> to keep a unified order of classes between different charts._
 
 """
-    + clickable_label
-)
 
 
-markdown_PR = (
-    """## Recall vs. Precision
+markdown_PR = """## Recall vs. Precision
 
 This section compares Precision and Recall in one graph, identifying **imbalance** between these two.
 
 _Bars in the chart are sorted by <abbr title="{}">F1-score</abbr> to keep a unified order of classes between different charts._
 
 """
-    + clickable_label
-)
 
 
 markdown_pr_curve = """## Precision-Recall Curve
@@ -199,44 +168,39 @@ notification_ap = {
     "description": "",
 }
 
-markdown_pr_by_class = (
-    """### Precision-Recall Curve by Class
+markdown_pr_by_class = """### Precision-Recall Curve by Class
 
 In this plot, you can evaluate PR curve for each class individually.
 
 """
-    + clickable_label
-)
 
-markdown_confusion_matrix = (
-    """## Confusion Matrix
+markdown_confusion_matrix = """## Confusion Matrix
 
 Confusion matrix helps to find the number of confusions between different classes made by the model.
 Each row of the matrix represents the instances in a ground truth class, while each column represents the instances in a predicted class.
 The diagonal elements represent the number of correct predictions for each class (True Positives), and the off-diagonal elements show misclassifications.
 
 """
-    + clickable_label
-)
 
+markdown_frequently_confused_empty = """### Frequently Confused Classes
+
+No frequently confused class pairs found
+"""
 
-markdown_frequently_confused = (
-    """### Frequently Confused Classes
+markdown_frequently_confused = """### Frequently Confused Classes
 
 This chart displays the most frequently confused pairs of classes. In general, it finds out which classes visually seem very similar to the model.
 
-The chart calculates the **probability of confusion** between different pairs of classes. For instance, if the probability of confusion for the pair “{} - {}” is {}, this means that when the model predicts either “{}” or “{}”, there is a {}% chance that the model might mistakenly predict one instead of the other.
+The chart calculates the **probability of confusion** between different pairs of classes. For instance, if the probability of confusion for the pair "{} - {}" is {}, this means that when the model predicts either "{}" or "{}", there is a {}% chance that the model might mistakenly predict one instead of the other.
 
 The measure is class-symmetric, meaning that the probability of confusing a {} with a {} is equal to the probability of confusing a {} with a {}.
 
 """
-    + clickable_label
-)
 
 
-markdown_localization_accuracy = """## Mask accuracy (IoU)
+markdown_localization_accuracy = """## Localization Accuracy (IoU)
 
-This section measures how accurately predicted masks match the actual shapes of ground truth instances. We calculate the average <abbr title="{}">IoU score</abbr> of predictions and visualize a histogram of IoU scores.
+This section measures how accurately predicted masks match the actual shapes of ground truth instances.
 """
 
 markdown_iou_calculation = """<img src='https://github.com/dataset-ninja/model-benchmark-template/assets/78355358/8d7c63d0-2f3b-4f3f-9fd8-c6383a4bfba4' alt='alt text' width='300' />
@@ -261,6 +225,7 @@ This section analyzes <abbr title="{}">confidence scores</abbr> (or predicted pr
 """
 
 markdown_what_is_calibration = """In some applications, it's crucial for a model not only to make accurate predictions but also to provide reliable **confidence levels**. A well-calibrated model aligns its confidence scores with the actual likelihood of predictions being correct. For example, if a model claims 90% confidence for predictions but they are correct only half the time, it is **overconfident**. Conversely, **underconfidence** occurs when a model assigns lower confidence scores than the actual likelihood of its predictions. In the context of autonomous driving, this might cause a vehicle to brake or slow down too frequently, reducing travel efficiency and potentially causing traffic issues."""
+
 markdown_calibration_score_2 = """To evaluate the calibration, we draw a <b>Reliability Diagram</b> and calculate <b>Expected Calibration Error</b> (ECE)."""
 
 markdown_reliability_diagram = """### Reliability Diagram
@@ -269,7 +234,7 @@ Reliability diagram, also known as a Calibration curve, helps in understanding w
 """
 
 markdown_calibration_curve_interpretation = """
-1. **The curve is above the perfect line (Underconfidence):** If the calibration curve is consistently above the perfect line, this indicates underconfidence. The model’s predictions are more correct than the confidence scores suggest. For example, if the model assigns 70% confidence to some predictions but, empirically, 90% of these predictions are correct, the model is underconfident.
+1. **The curve is above the perfect line (Underconfidence):** If the calibration curve is consistently above the perfect line, this indicates underconfidence. The model's predictions are more correct than the confidence scores suggest. For example, if the model assigns 70% confidence to some predictions but, empirically, 90% of these predictions are correct, the model is underconfident.
 2. **The curve is below the perfect line (Overconfidence):** If the calibration curve is below the perfect line, the model exhibits overconfidence. This means it is too sure of its predictions. For example, if the model assigns 80% confidence to some predictions, but only 40% of these predictions are correct, the model is overconfident.
 
 To quantify the calibration, we calculate **Expected Calibration Error (ECE).** Intuitively, ECE can be viewed as a deviation of the model's calibration curve from the diagonal line, that corresponds to a perfectly calibrated model. When ECE is high, we can not trust predicted probabilities so much.
@@ -316,14 +281,11 @@ Additionally, it provides a view of how predicted probabilities are distributed.
 Ideally, the green histogram (TP predictions) should have higher confidence scores and be shifted to the right, indicating that the model is sure about its correct predictions, and the red histogram (FP predictions) should have lower confidence scores and be shifted to the left.
 """
 
-markdown_class_ap = (
-    """## Average Precision by Class
+markdown_class_ap = """## Average Precision by Class
 
 A quick visual comparison of the model performance across all classes. Each axis in the chart represents a different class, and the distance to the center indicates the <abbr title="{}">Average Precision</abbr> (AP) for that class.
 
 """
-    + clickable_label
-)
 
 
 markdown_class_outcome_counts_1 = """### Outcome Counts by Class
@@ -337,16 +299,38 @@ markdown_normalization = """Normalization is used for better interclass comparis
 If normalization is off, the chart will display the total count of instances that correspond to outcome type (one of TP, FP or FN). This mode is identical to the main Outcome Counts graph on the top of the page. However, when normalization is off, you may encounter a class imbalance problem. Visually, bars that correspond to classes with many instances in the dataset will be much larger than others. This complicates the visual analysis.
 """
 
-markdown_class_outcome_counts_2 = (
-    """You can switch the plot view between normalized and absolute values.
+markdown_class_outcome_counts_2 = """You can switch the plot view between normalized and absolute values.
 
 _Bars in the chart are sorted by <abbr title="{}">F1-score</abbr> to keep a unified order of classes between different charts._
 
 """
-    + clickable_label
-)
 
 empty = """### {}
 
 > {}
 """
+
+
+markdown_speedtest_intro = """## Inference Speed
+
+This is a speed test benchmark for this model. The model was tested with the following configuration:
+
+- **Device**: {}
+- **Hardware**: {}
+- **Runtime**: {}
+"""
+
+markdown_speedtest_table = """
+The table below shows the speed test results. For each test, the time taken to process one batch of images is shown, as well as the model's throughput (i.e, the number of images processed per second, or FPS). Results are averaged across **{}** iterations.
+"""
+
+markdown_real_time_inference = """## Real-time Inference
+
+This chart compares different runtimes and devices (CPU or GPU)."""
+
+# We additionally divide **predict** procedure into three stages: pre-process, inference, and post-process. Each bar in this chart consists of these three stages. For example, in the chart you can find how long the post-process phase lasts in a CPU device with an ONNXRuntime environment."""
+
+
+markdown_speedtest_chart = """
+This chart shows how the model's speed changes with different batch sizes . As the batch size increases, you can observe an increase in FPS (images per second).
+"""

supervisely/nn/benchmark/instance_segmentation/visualizer.py

@@ -0,0 +1,18 @@
+import supervisely.nn.benchmark.instance_segmentation.text_templates as vis_texts
+from supervisely.nn.benchmark.cv_tasks import CVTask
+from supervisely.nn.benchmark.object_detection.visualizer import (
+    ObjectDetectionVisualizer,
+)
+
+
+class InstanceSegmentationVisualizer(ObjectDetectionVisualizer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.vis_texts = vis_texts
+        self._widgets = False
+        self.ann_opacity = 0.7
+
+    @property
+    def cv_task(self):
+        return CVTask.INSTANCE_SEGMENTATION

supervisely/nn/benchmark/object_detection/base_vis_metric.py

@@ -0,0 +1,51 @@
+from typing import Dict, Optional
+
+from supervisely.nn.benchmark.base_visualizer import BaseVisMetric
+from supervisely.nn.benchmark.object_detection.evaluator import (
+    ObjectDetectionEvalResult,
+)
+
+
+class DetectionVisMetric(BaseVisMetric):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.eval_result: ObjectDetectionEvalResult
+
+    def get_click_data(self) -> Optional[Dict]:
+        if not self.clickable:
+            return
+
+        res = {}
+
+        res["layoutTemplate"] = [None, None, None]
+        res["clickData"] = {}
+        for key, v in self.eval_result.click_data.objects_by_class.items():
+            res["clickData"][key] = {}
+            res["clickData"][key]["imagesIds"] = []
+
+            img_ids = set()
+            obj_ids = set()
+
+            res["clickData"][key][
+                "title"
+            ] = f"{key} class: {len(v)} object{'s' if len(v) > 1 else ''}"
+
+            for x in v:
+                img_ids.add(x["dt_img_id"])
+                obj_id = x["dt_obj_id"]
+                if obj_id is not None:
+                    obj_ids.add(obj_id)
+
+            res["clickData"][key]["imagesIds"] = list(img_ids)
+            res["clickData"][key]["filters"] = [
+                {
+                    "type": "tag",
+                    "tagId": "confidence",
+                    "value": [self.eval_result.mp.f1_optimal_conf, 1],
+                },
+                {"type": "tag", "tagId": "outcome", "value": "TP"},
+                {"type": "specific_objects", "tagId": None, "value": list(obj_ids)},
+            ]
+
+        return res

supervisely/nn/benchmark/{object_detection_benchmark.py → object_detection/benchmark.py}

@@ -1,7 +1,8 @@
 from supervisely.nn.benchmark.base_benchmark import BaseBenchmark
 from supervisely.nn.benchmark.cv_tasks import CVTask
-from supervisely.nn.benchmark.evaluation.object_detection_evaluator import (
-    ObjectDetectionEvaluator,
+from supervisely.nn.benchmark.object_detection.evaluator import ObjectDetectionEvaluator
+from supervisely.nn.benchmark.object_detection.visualizer import (
+    ObjectDetectionVisualizer,
 )
 from supervisely.nn.benchmark.utils import try_set_conf_auto
 
@@ -9,6 +10,7 @@ CONF_THRES = 0.05
 
 
 class ObjectDetectionBenchmark(BaseBenchmark):
+    visualizer_cls = ObjectDetectionVisualizer
 
     @property
     def cv_task(self) -> str:

supervisely/nn/benchmark/object_detection/evaluation_params.yaml

@@ -0,0 +1,2 @@
+# Intersection over Union threshold that will be used for objects mathcing
+iou_threshold: 0.5

supervisely/nn/benchmark/{evaluation/object_detection_evaluator.py → object_detection/evaluator.py}

@@ -1,14 +1,72 @@
 import os
-
-from supervisely.io.json import dump_json_file
-from supervisely.nn.benchmark.coco_utils import read_coco_datasets, sly2coco
-from supervisely.nn.benchmark.evaluation import BaseEvaluator
-from supervisely.nn.benchmark.evaluation.coco import calculate_metrics
+import pickle
 from pathlib import Path
 
+import pandas as pd
+
+from supervisely.io.json import dump_json_file, load_json_file
+from supervisely.nn.benchmark.base_evaluator import BaseEvalResult, BaseEvaluator
+from supervisely.nn.benchmark.object_detection.metric_provider import MetricProvider
+from supervisely.nn.benchmark.utils import (
+    calculate_metrics,
+    read_coco_datasets,
+    sly2coco,
+)
+from supervisely.nn.benchmark.visualization.vis_click_data import ClickData, IdMapper
+
+
+class ObjectDetectionEvalResult(BaseEvalResult):
+    mp_cls = MetricProvider
+
+    def _read_eval_data(self):
+        from pycocotools.coco import COCO  # pylint: disable=import-error
+
+        gt_path = str(Path(self.directory) / "cocoGt.json")
+        dt_path = str(Path(self.directory) / "cocoDt.json")
+        coco_gt, coco_dt = COCO(gt_path), COCO(dt_path)
+        self.coco_gt = coco_gt
+        self.coco_dt = coco_dt
+        self.eval_data = None
+        with open(Path(self.directory, "eval_data.pkl"), "rb") as f:
+            self.eval_data = pickle.load(f)
+
+        inference_info_path = Path(self.directory) / "inference_info.json"
+        self.inference_info = load_json_file(str(inference_info_path))
+
+        speedtest_info_path = Path(self.directory).parent / "speedtest" / "speedtest.json"
+        if speedtest_info_path.exists():
+            self.speedtest_info = load_json_file(str(speedtest_info_path))
+
+        self.mp = MetricProvider(
+            self.eval_data["matches"],
+            self.eval_data["coco_metrics"],
+            self.eval_data["params"],
+            self.coco_gt,
+            self.coco_dt,
+        )
+        self.mp.calculate()
+
+        self.df_score_profile = pd.DataFrame(
+            self.mp.confidence_score_profile(), columns=["scores", "precision", "recall", "f1"]
+        )
+
+        # downsample
+        if len(self.df_score_profile) > 5000:
+            self.dfsp_down = self.df_score_profile.iloc[:: len(self.df_score_profile) // 1000]
+        else:
+            self.dfsp_down = self.df_score_profile
+
+        # Click data
+        gt_id_mapper = IdMapper(self.coco_gt.dataset)
+        dt_id_mapper = IdMapper(self.coco_dt.dataset)
+
+        self.click_data = ClickData(self.mp.m, gt_id_mapper, dt_id_mapper)
+
 
 class ObjectDetectionEvaluator(BaseEvaluator):
-    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/coco/evaluation_params.yaml"
+    EVALUATION_PARAMS_YAML_PATH = f"{Path(__file__).parent}/evaluation_params.yaml"
+    eval_result_cls = ObjectDetectionEvalResult
+    accepted_shapes = ["rectangle"]
 
     def evaluate(self):
         try:
@@ -43,14 +101,14 @@ class ObjectDetectionEvaluator(BaseEvaluator):
         cocoGt_json = sly2coco(
             self.gt_project_path,
             is_dt_dataset=False,
-            accepted_shapes=["rectangle"],
+            accepted_shapes=self.accepted_shapes,
             progress=self.pbar,
             classes_whitelist=self.classes_whitelist,
         )
         cocoDt_json = sly2coco(
-            self.dt_project_path,
+            self.pred_project_path,
             is_dt_dataset=True,
-            accepted_shapes=["rectangle"],
+            accepted_shapes=self.accepted_shapes,
             progress=self.pbar,
             classes_whitelist=self.classes_whitelist,
         )

supervisely/nn/benchmark/{evaluation/coco → object_detection}/metric_provider.py

@@ -4,7 +4,7 @@ from copy import deepcopy
 import numpy as np
 import pandas as pd
 
-from supervisely.nn.benchmark.evaluation.coco import metrics
+from supervisely.nn.benchmark.utils.detection import metrics
 
 METRIC_NAMES = {
     "mAP": "mAP",
@@ -266,12 +266,12 @@ class _MetricProvider:
     def _init_counts(self):
         cat_ids = self.cat_ids
         iouThrs = self.iouThrs
-        catId2idx = {cat_id: idx for idx, cat_id in enumerate(cat_ids)}
+        cat_id_to_idx = {cat_id: idx for idx, cat_id in enumerate(cat_ids)}
         ious = []
         cats = []
         for match in self.tp_matches:
             ious.append(match["iou"])
-            cats.append(catId2idx[match["category_id"]])
+            cats.append(cat_id_to_idx[match["category_id"]])
         ious = np.array(ious) + np.spacing(1)
         iou_idxs = np.searchsorted(iouThrs, ious) - 1
         cats = np.array(cats)
@@ -285,7 +285,7 @@ class _MetricProvider:
         true_positives = true_positives[:, ::-1].cumsum(1)[:, ::-1]
         tp_count = true_positives[:, 0]
         # FN
-        cats_fn = np.array([catId2idx[match["category_id"]] for match in self.fn_matches])
+        cats_fn = np.array([cat_id_to_idx[match["category_id"]] for match in self.fn_matches])
         if cats_fn.size == 0:
             fn_count = np.zeros((len(cat_ids),), dtype=int)
         else:
@@ -293,7 +293,7 @@ class _MetricProvider:
         gt_count = fn_count + tp_count
         false_negatives = gt_count[:, None] - true_positives
         # FP
-        cats_fp = np.array([catId2idx[match["category_id"]] for match in self.fp_matches])
+        cats_fp = np.array([cat_id_to_idx[match["category_id"]] for match in self.fp_matches])
         if cats_fp.size == 0:
             fp_count = np.zeros((len(cat_ids),), dtype=int)
         else:
@@ -380,33 +380,32 @@ class _MetricProvider:
 
     def confusion_matrix(self):
         K = len(self.cat_ids)
-        catId2idx = {cat_id: i for i, cat_id in enumerate(self.cat_ids)}
-        idx2catId = {i: cat_id for cat_id, i in catId2idx.items()}
+        cat_id_to_idx = {cat_id: i for i, cat_id in enumerate(self.cat_ids)}
 
         confusion_matrix = np.zeros((K + 1, K + 1), dtype=int)
 
         for m in self.confused_matches:
-            cat_idx_pred = catId2idx[m["category_id"]]
-            cat_idx_gt = catId2idx[self.cocoGt.anns[m["gt_id"]]["category_id"]]
+            cat_idx_pred = cat_id_to_idx[m["category_id"]]
+            cat_idx_gt = cat_id_to_idx[self.cocoGt.anns[m["gt_id"]]["category_id"]]
             confusion_matrix[cat_idx_pred, cat_idx_gt] += 1
 
         for m in self.tp_matches:
-            cat_idx = catId2idx[m["category_id"]]
+            cat_idx = cat_id_to_idx[m["category_id"]]
             confusion_matrix[cat_idx, cat_idx] += 1
 
         for m in self.fp_not_confused_matches:
-            cat_idx_pred = catId2idx[m["category_id"]]
+            cat_idx_pred = cat_id_to_idx[m["category_id"]]
             confusion_matrix[cat_idx_pred, -1] += 1
 
         for m in self.fn_matches:
-            cat_idx_gt = catId2idx[m["category_id"]]
+            cat_idx_gt = cat_id_to_idx[m["category_id"]]
             confusion_matrix[-1, cat_idx_gt] += 1
 
         return confusion_matrix
 
     def frequently_confused(self, confusion_matrix, topk_pairs=20):
         # Frequently confused class pairs
-        idx2catId = {i: cat_id for i, cat_id in enumerate(self.cat_ids)}
+        cat_id_enum = {i: cat_id for i, cat_id in enumerate(self.cat_ids)}
         cm = confusion_matrix[:-1, :-1]
         cm_l = np.tril(cm, -1)
         cm_u = np.triu(cm, 1)
@@ -427,7 +426,7 @@ class _MetricProvider:
         confused_name_pairs = [(self.cat_names[i], self.cat_names[j]) for i, j in confused_idxs]
         confused_counts = confused_counts[inds_sort2]
         confused_prob = confused_prob[inds_sort2]
-        confused_catIds = [(idx2catId[i], idx2catId[j]) for i, j in confused_idxs]
+        confused_catIds = [(cat_id_enum[i], cat_id_enum[j]) for i, j in confused_idxs]
 
         return pd.DataFrame(
             {