faster-coco-eval 1.5.4__tar.gz → 1.5.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {faster_coco_eval-1.5.4/faster_coco_eval.egg-info → faster_coco_eval-1.5.5}/PKG-INFO +6 -2
  2. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/README.md +5 -1
  3. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/core/coco.py +71 -1
  4. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/core/cocoeval.py +7 -0
  5. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/core/faster_eval_api.py +41 -72
  6. faster_coco_eval-1.5.5/faster_coco_eval/extra/curves.py +158 -0
  7. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/extra/display.py +25 -1
  8. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/extra/draw.py +121 -4
  9. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/extra/extra.py +21 -2
  10. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/version.py +1 -1
  11. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5/faster_coco_eval.egg-info}/PKG-INFO +6 -2
  12. faster_coco_eval-1.5.4/faster_coco_eval/extra/curves.py +0 -63
  13. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/LICENSE +0 -0
  14. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/MANIFEST.in +0 -0
  15. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/csrc/faster_eval_api/coco_eval/cocoeval.cpp +0 -0
  16. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/csrc/faster_eval_api/coco_eval/cocoeval.h +0 -0
  17. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/csrc/faster_eval_api/faster_eval_api.cpp +0 -0
  18. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/csrc/mask/common/maskApi.c +0 -0
  19. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/csrc/mask/common/maskApi.h +0 -0
  20. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/csrc/mask/pycocotools/_mask.pyx +0 -0
  21. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/__init__.py +0 -0
  22. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/core/__init__.py +0 -0
  23. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/core/mask.py +0 -0
  24. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/extra/__init__.py +0 -0
  25. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval/extra/utils.py +0 -0
  26. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval.egg-info/SOURCES.txt +0 -0
  27. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval.egg-info/dependency_links.txt +0 -0
  28. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval.egg-info/requires.txt +0 -0
  29. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/faster_coco_eval.egg-info/top_level.txt +0 -0
  30. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/pyproject.toml +0 -0
  31. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/requirements/extra.txt +0 -0
  32. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/requirements/runtime.txt +0 -0
  33. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/setup.cfg +0 -0
  34. {faster_coco_eval-1.5.4 → faster_coco_eval-1.5.5}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: faster-coco-eval
3
- Version: 1.5.4
3
+ Version: 1.5.5
4
4
  Summary: Faster interpretation of the original COCOEval
5
5
  Home-page: https://github.com/MiXaiLL76/faster_coco_eval
6
6
  Author: MiXaiLL76
@@ -53,7 +53,7 @@ For our use case with a test dataset of 5000 images from the coco val dataset.
53
53
  Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
54
54
 
55
55
  Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://github.com/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
56
- [colab_example.ipynb in google collab](https://drive.google.com/file/d/13raC0nRsil797ITJSv3FTRvUvfEK9SD-/view?usp=sharing)
56
+ [colab_example.ipynb in google collab](https://colab.research.google.com/drive/1qj392oIU8fmeyIFHCtxCrLA8PQQAMoIs)
57
57
  Tested with rtmdet model bbox + segm
58
58
 
59
59
  ### Summary for 5000 imgs
@@ -76,6 +76,10 @@ Code examples for using the library are available on the [Wiki](https://github.c
76
76
 
77
77
  Available via link [history.md](https://github.com/MiXaiLL76/faster_coco_eval/blob/main/history.md)
78
78
 
79
+ ## Star History
80
+
81
+ [![Star History Chart](https://api.star-history.com/svg?repos=MiXaiLL76/faster_coco_eval&type=Date)](https://star-history.com/#MiXaiLL76/faster_coco_eval&Date)
82
+
79
83
  ## License
80
84
 
81
85
  The original module was licensed with apache 2, I will continue with the same license.
@@ -28,7 +28,7 @@ For our use case with a test dataset of 5000 images from the coco val dataset.
28
28
  Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
29
29
 
30
30
  Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://github.com/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
31
- [colab_example.ipynb in google collab](https://drive.google.com/file/d/13raC0nRsil797ITJSv3FTRvUvfEK9SD-/view?usp=sharing)
31
+ [colab_example.ipynb in google collab](https://colab.research.google.com/drive/1qj392oIU8fmeyIFHCtxCrLA8PQQAMoIs)
32
32
  Tested with rtmdet model bbox + segm
33
33
 
34
34
  ### Summary for 5000 imgs
@@ -51,6 +51,10 @@ Code examples for using the library are available on the [Wiki](https://github.c
51
51
 
52
52
  Available via link [history.md](https://github.com/MiXaiLL76/faster_coco_eval/blob/main/history.md)
53
53
 
54
+ ## Star History
55
+
56
+ [![Star History Chart](https://api.star-history.com/svg?repos=MiXaiLL76/faster_coco_eval&type=Date)](https://star-history.com/#MiXaiLL76/faster_coco_eval&Date)
57
+
54
58
  ## License
55
59
 
56
60
  The original module was licensed with apache 2, I will continue with the same license.
@@ -105,8 +105,11 @@ class COCO:
105
105
  def createIndex(self):
106
106
  # create index
107
107
  logger.debug("creating index...")
108
- anns, cats, imgs = {}, {}, {}
108
+ anns, cats, imgs, annToImgs = {}, {}, {}, {}
109
109
  imgToAnns, catToImgs = defaultdict(list), defaultdict(list)
110
+ imgCatToAnnsIdx = defaultdict(dict)
111
+ imgToAnnsIdx = defaultdict(dict)
112
+
110
113
  annsImgIds_dict = {}
111
114
  if "images" in self.dataset:
112
115
  for img in self.dataset["images"]:
@@ -120,6 +123,15 @@ class COCO:
120
123
  if annsImgIds_dict.get(ann["image_id"]):
121
124
  imgToAnns[ann["image_id"]].append(ann)
122
125
  anns[ann["id"]] = ann
126
+ annToImgs[ann["id"]] = ann["image_id"]
127
+ imgCatToAnnsIdx[(ann["image_id"], ann["category_id"])][
128
+ ann["id"]
129
+ ] = len(
130
+ imgCatToAnnsIdx[(ann["image_id"], ann["category_id"])]
131
+ )
132
+ imgToAnnsIdx[ann["image_id"]][ann["id"]] = len(
133
+ imgToAnnsIdx[ann["image_id"]]
134
+ )
123
135
 
124
136
  if "categories" in self.dataset:
125
137
  for cat in self.dataset["categories"]:
@@ -135,6 +147,9 @@ class COCO:
135
147
  self.anns = anns
136
148
  self.imgToAnns = imgToAnns
137
149
  self.catToImgs = catToImgs
150
+ self.annToImgs = annToImgs
151
+ self.imgCatToAnnsIdx = imgCatToAnnsIdx
152
+ self.imgToAnnsIdx = imgToAnnsIdx
138
153
  self.imgs = imgs
139
154
  self.cats = cats
140
155
 
@@ -453,21 +468,64 @@ class COCO:
453
468
  return m
454
469
 
455
470
  def get_ann_ids(self, img_ids=[], cat_ids=[], area_rng=[], iscrowd=None):
471
+ """Get ann ids that satisfy given filter conditions.
472
+
473
+ :param img_ids (int array) : get anns for given imgs
474
+ :param cat_ids (int array) : get anns for given cats
475
+ :param area_rng (float array) : get anns for given area range
476
+ (e.g. [0 inf])
477
+ :return: ids (int array) : integer array of ann ids
478
+
479
+ """
456
480
  return self.getAnnIds(img_ids, cat_ids, area_rng, iscrowd)
457
481
 
458
482
  def get_cat_ids(self, cat_names=[], sup_names=[], cat_ids=[]):
483
+ """Get cat ids that satisfy given filter conditions.
484
+
485
+ :param cat_names (str array) : get cats for given cat names
486
+ :param sup_names (str array) : get cats for given supercategory
487
+ names
488
+ :param cat_ids (int array) : get cats for given cat ids
489
+ :return: ids (int array) : integer array of cat ids
490
+
491
+ """
459
492
  return self.getCatIds(cat_names, sup_names, cat_ids)
460
493
 
461
494
  def get_img_ids(self, img_ids=[], cat_ids=[]):
495
+ """Get img ids that satisfy given filter conditions.
496
+
497
+ :param img_ids (int array) : get imgs for given ids
498
+ :param cat_ids (int array) : get imgs with all given cats
499
+ :return: ids (int array) : integer array of img ids
500
+
501
+ """
462
502
  return self.getImgIds(img_ids, cat_ids)
463
503
 
464
504
  def load_anns(self, ids):
505
+ """Load anns with the specified ids.
506
+
507
+ :param ids (int array) : integer ids specifying anns
508
+ :return: anns (object array) : loaded ann objects
509
+
510
+ """
465
511
  return self.loadAnns(ids)
466
512
 
467
513
  def load_cats(self, ids):
514
+ """Load cats with the specified ids.
515
+
516
+ :param ids (int array) : integer ids specifying cats
517
+ :return: cats (object array) : loaded cat objects
518
+
519
+ """
468
520
  return self.loadCats(ids)
469
521
 
470
522
  def load_imgs(self, ids):
523
+ """Load anns with the specified ids.
524
+
525
+ :param ids (int array) : integer ids specifying img
526
+ :return: imgs (object array) : loaded img objects
527
+
528
+ """
471
529
  return self.loadImgs(ids)
472
530
 
473
531
  @property
@@ -477,3 +535,15 @@ class COCO:
477
535
  @property
478
536
  def cat_img_map(self):
479
537
  return self.catToImgs
538
+
539
+ @property
540
+ def ann_img_map(self):
541
+ return self.annToImgs
542
+
543
+ @property
544
+ def img_ann_idx_map(self):
545
+ return self.imgToAnnsIdx
546
+
547
+ @property
548
+ def img_cat_ann_idx_map(self):
549
+ return self.imgCatToAnnsIdx
@@ -592,6 +592,13 @@ class Params:
592
592
  )
593
593
 
594
594
  def __init__(self, iouType="segm", kpt_sigmas=None):
595
+ """Params for coco evaluation api.
596
+
597
+ IouType: the type of iou to use for evaluation, can be 'segm', 'bbox',
598
+ or 'keypoints'
599
+ kpt_sigmas: list of keypoint sigma values.
600
+
601
+ """
595
602
  if iouType == "segm" or iouType == "bbox":
596
603
  self.setDetParams()
597
604
  elif iouType == "keypoints":
@@ -9,7 +9,6 @@ import numpy as np
9
9
 
10
10
  import faster_coco_eval.faster_eval_api_cpp as _C
11
11
 
12
- from . import mask as maskUtils
13
12
  from .cocoeval import COCOeval
14
13
 
15
14
  logger = logging.getLogger(__name__)
@@ -157,16 +156,21 @@ class COCOeval_faster(COCOeval):
157
156
  self.matched = False
158
157
  try:
159
158
  if self.extra_calc:
159
+
160
+ num_iou_thresholds, _, _, num_area_ranges, _ = self.eval[
161
+ "counts"
162
+ ]
163
+
160
164
  self.detection_matches = np.vstack(
161
165
  np.array(self.eval["detection_matches"]).reshape(
162
- self.eval["counts"][0], self.eval["counts"][3], -1
166
+ num_iou_thresholds, num_area_ranges, -1
163
167
  )
164
168
  )
165
169
  assert self.detection_matches.shape[1] <= len(self.cocoDt.anns)
166
170
 
167
171
  self.ground_truth_matches = np.vstack(
168
172
  np.array(self.eval["ground_truth_matches"]).reshape(
169
- self.eval["counts"][0], self.eval["counts"][3], -1
173
+ num_iou_thresholds, num_area_ranges, -1
170
174
  )
171
175
  )
172
176
  assert self.ground_truth_matches.shape[1] <= len(
@@ -175,7 +179,7 @@ class COCOeval_faster(COCOeval):
175
179
 
176
180
  self.ground_truth_orig_id = np.vstack(
177
181
  np.array(self.eval["ground_truth_orig_id"]).reshape(
178
- self.eval["counts"][0], self.eval["counts"][3], -1
182
+ num_iou_thresholds, num_area_ranges, -1
179
183
  )
180
184
  )
181
185
  assert self.ground_truth_orig_id.shape[1] <= len(
@@ -192,6 +196,8 @@ class COCOeval_faster(COCOeval):
192
196
  self.print_function("DONE (t={:0.2f}s).".format(toc - tic))
193
197
 
194
198
  def math_matches(self):
199
+ """For each ground truth, find the best matching detection and set the
200
+ detection as matched."""
195
201
  for gidx, ground_truth_matches in enumerate(self.ground_truth_matches):
196
202
  gt_ids = self.ground_truth_orig_id[gidx]
197
203
 
@@ -203,24 +209,29 @@ class COCOeval_faster(COCOeval):
203
209
  if gt_id <= -1:
204
210
  continue
205
211
 
206
- _gt_ann = self.cocoGt.anns.get(gt_id)
207
- if _gt_ann is None:
208
- continue
212
+ _gt_ann = self.cocoGt.anns[gt_id]
213
+ _dt_ann = self.cocoDt.anns[dt_id]
214
+ _img_id = self.cocoGt.ann_img_map[gt_id]
215
+ _catId = _gt_ann["category_id"] if self.params.useCats else -1
209
216
 
210
- _dt_ann = self.cocoDt.anns.get(dt_id)
211
- if _dt_ann is None:
212
- continue
217
+ if self.params.useCats:
218
+ _catId = _gt_ann["category_id"]
219
+ _map_gt_dict = self.cocoGt.img_cat_ann_idx_map
220
+ _map_dt_dict = self.cocoDt.img_cat_ann_idx_map
221
+ _map_id = (_img_id, _catId)
222
+ else:
223
+ _catId = -1
224
+ _map_gt_dict = self.cocoGt.img_ann_idx_map
225
+ _map_dt_dict = self.cocoDt.img_ann_idx_map
226
+ _map_id = _img_id
213
227
 
214
- if int(_gt_ann["image_id"]) != int(_dt_ann["image_id"]):
215
- continue
228
+ iou_gt_id = _map_gt_dict[_map_id].get(gt_id)
229
+ iou_dt_id = _map_dt_dict[_map_id].get(dt_id)
216
230
 
217
- if self.params.useCats == 1:
218
- if int(_gt_ann["category_id"]) != int(
219
- _dt_ann["category_id"]
220
- ):
221
- continue
231
+ if iou_gt_id is None or iou_dt_id is None:
232
+ continue
222
233
 
223
- iou = self.computeAnnIoU(_gt_ann, _dt_ann)
234
+ iou = self.ious[(_img_id, _catId)][iou_dt_id, iou_gt_id]
224
235
 
225
236
  if not _gt_ann.get("matched", False):
226
237
  _dt_ann["tp"] = True
@@ -251,63 +262,16 @@ class COCOeval_faster(COCOeval):
251
262
  if self.cocoGt.anns[gt_id].get("matched") is None:
252
263
  self.cocoGt.anns[gt_id]["fn"] = True
253
264
 
254
- def computeAnnIoU(self, gt_ann, dt_ann):
255
- g = []
256
- d = []
257
-
258
- if self.params.iouType == "segm":
259
- g.append(gt_ann["rle"])
260
- d.append(dt_ann["rle"])
261
- elif self.params.iouType == "bbox":
262
- g.append(gt_ann["bbox"])
263
- d.append(dt_ann["bbox"])
264
-
265
- iscrowd = [0 for _ in g]
266
-
267
- _iou = maskUtils.iou(d, g, iscrowd)
268
-
269
- if len(_iou) == 0:
270
- return 0
271
- else:
272
- return _iou.max()
273
-
274
- def compute_mIoU(self, categories=None, raw=False):
275
- if self.params.iouType == "keypoints":
276
- return np.array(
277
- [val.max() for val in self.ious.values() if len(val)]
278
- ).mean()
279
-
280
- g = []
281
- d = []
282
- s = []
283
-
265
+ def compute_mIoU(self):
266
+ """Compute the mIoU metric."""
267
+ ious = []
284
268
  for _, dt_ann in self.cocoDt.anns.items():
285
- if dt_ann.get("tp", False):
286
- gt_ann = self.cocoGt.anns[dt_ann["gt_id"]]
287
- if categories is None or gt_ann["category_id"] in categories:
288
- s.append(dt_ann.get("score", 1))
289
- if self.params.iouType == "segm":
290
- g.append(gt_ann["rle"])
291
- d.append(dt_ann["rle"])
292
- elif self.params.iouType == "bbox":
293
- g.append(gt_ann["bbox"])
294
- d.append(dt_ann["bbox"])
295
- else:
296
- raise Exception("unknown iouType for iou computation")
297
-
298
- iscrowd = [0 for _ in g]
299
-
300
- ious = maskUtils.iou(d, g, iscrowd)
301
- if raw:
302
- return ious
303
-
304
- if len(ious) == 0:
305
- return 0
306
- else:
307
- ious = ious.diagonal()
308
- return ious.mean()
269
+ if dt_ann.get("iou", False):
270
+ ious.append(dt_ann["iou"])
271
+ return sum(ious) / len(ious)
309
272
 
310
273
  def compute_mAUC(self):
274
+ """Compute the mAUC metric."""
311
275
  aucs = []
312
276
 
313
277
  for K in range(self.eval["counts"][2]):
@@ -376,6 +340,11 @@ class COCOeval_faster(COCOeval):
376
340
 
377
341
  @staticmethod
378
342
  def calc_auc(recall_list, precision_list):
343
+ """
344
+ Calculate area under precision recall curve
345
+ recall_list: list of recall values
346
+ precision_list: list of precision values
347
+ """
379
348
  # https://towardsdatascience.com/how-to-efficiently-implement-area-under-precision-recall-curve-pr-auc-a85872fd7f14
380
349
  # mrec = np.concatenate(([0.], recall_list, [1.]))
381
350
  # mpre = np.concatenate(([0.], precision_list, [0.]))
@@ -0,0 +1,158 @@
1
+ import logging
2
+
3
+ import numpy as np
4
+
5
+ from ..core import COCOeval_faster
6
+ from .draw import plot_ced_metric, plot_f1_confidence, plot_pre_rec
7
+ from .extra import ExtraEval
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ class Curves(ExtraEval):
13
+ def build_curve(self, label: str):
14
+ """Build the curve for a given label."""
15
+ assert self.eval is not None, "Run first self.evaluate()"
16
+
17
+ curve = []
18
+
19
+ if self.useCats:
20
+ cat_ids = list(range(self.eval["precision"].shape[2]))
21
+ else:
22
+ cat_ids = [0]
23
+
24
+ for category_id in cat_ids:
25
+ _label = "[{}={}] ".format(label, category_id)
26
+ if len(cat_ids) == 1:
27
+ _label = ""
28
+
29
+ precision_list = self.eval["precision"][
30
+ :, :, category_id, :, :
31
+ ].ravel()
32
+ recall_list = self.recThrs
33
+ scores = self.eval["scores"][:, :, category_id, :, :].ravel()
34
+ auc = round(
35
+ COCOeval_faster.calc_auc(recall_list, precision_list), 4
36
+ )
37
+
38
+ curve.append(
39
+ dict(
40
+ recall_list=recall_list,
41
+ precision_list=precision_list,
42
+ name="{}auc: {:.3f}".format(_label, auc),
43
+ label=_label,
44
+ scores=scores,
45
+ auc=auc,
46
+ category_id=category_id,
47
+ )
48
+ )
49
+
50
+ return curve
51
+
52
+ def plot_pre_rec(
53
+ self, curves=None, label: str = "category_id", return_fig: bool = False
54
+ ):
55
+ """Plot the precision-recall curve.
56
+
57
+ curves: list of curves to plot
58
+ label: label of the curves
59
+ return_fig: return the figure
60
+
61
+ """
62
+ if curves is None:
63
+ curves = self.build_curve(label)
64
+
65
+ return plot_pre_rec(curves, return_fig=return_fig)
66
+
67
+ def plot_f1_confidence(
68
+ self, curves=None, label: str = "category_id", return_fig: bool = False
69
+ ):
70
+ """Plot the F1 confidence curve.
71
+
72
+ curves: list of curves to plot
73
+ label: label of the curves
74
+ return_fig: return the figure
75
+
76
+ """
77
+ if curves is None:
78
+ curves = self.build_curve(label)
79
+
80
+ return plot_f1_confidence(curves, return_fig=return_fig)
81
+
82
+ def build_ced_curve(self, mse_count: int = 1000):
83
+ """Build the curve for all categories."""
84
+ assert self.eval is not None, "Run first self.evaluate()"
85
+
86
+ curves = []
87
+ for category_id, category in self.cocoGt.cats.items():
88
+ all_mse = []
89
+ for ann_id in self.cocoGt.get_ann_ids(cat_ids=[category_id]):
90
+ gt_ann = self.cocoGt.anns[ann_id]
91
+ if gt_ann.get("keypoints", False) and gt_ann.get(
92
+ "matched", False
93
+ ):
94
+ dt_ann = self.cocoDt.anns[gt_ann["dt_id"]]
95
+
96
+ # https://en.wikipedia.org/wiki/Mean_squared_error
97
+ if self.iouType == "keypoints":
98
+ gt_kps = np.array(gt_ann["keypoints"]).reshape(-1, 3)[
99
+ :, :2
100
+ ]
101
+ dt_kps = np.array(dt_ann["keypoints"]).reshape(-1, 3)[
102
+ :, :2
103
+ ]
104
+ elif self.iouType == "bbox":
105
+ gt_kps = np.array(gt_ann["bbox"]) # xywh
106
+ gt_kps[2:] += gt_kps[:2] # xyxy
107
+ dt_kps = np.array(dt_ann["bbox"]) # xywh
108
+ dt_kps[2:] += dt_kps[:2] # xyxy
109
+ else:
110
+ raise ValueError(
111
+ f"not supported iouType {self.iouType} for CED"
112
+ )
113
+
114
+ mse = np.square(gt_kps - dt_kps).mean()
115
+ all_mse.append(mse)
116
+
117
+ if len(all_mse) == 0:
118
+ continue
119
+
120
+ all_mse = np.array(all_mse)
121
+ _median = np.median(all_mse)
122
+ _q3 = np.sqrt(np.var(all_mse))
123
+
124
+ _curve = {
125
+ "mse": [0],
126
+ "count": [0],
127
+ "total_count": len(all_mse),
128
+ "category": category,
129
+ }
130
+
131
+ for max_mse in np.linspace(
132
+ all_mse.min(), (_median + _q3), mse_count
133
+ ):
134
+ _mask = all_mse < max_mse
135
+ _curve["count"].append(_mask.sum())
136
+ _curve["mse"].append(max_mse)
137
+
138
+ _curve["count"].append(_curve["total_count"])
139
+ _curve["mse"].append(all_mse.max())
140
+ curves.append(_curve)
141
+ return curves
142
+
143
+ def plot_ced_metric(
144
+ self, curves=None, normalize: bool = True, return_fig: bool = False
145
+ ):
146
+ """Plot the CED metric curve.
147
+
148
+ curves: list of curves to plot
149
+ normalize: normalize the curve
150
+ return_fig: return the figure
151
+
152
+ """
153
+ if curves is None:
154
+ curves = self.build_ced_curve()
155
+
156
+ return plot_ced_metric(
157
+ curves, normalize=normalize, return_fig=return_fig
158
+ )
@@ -21,6 +21,17 @@ class PreviewResults(ExtraEval):
21
21
  categories: Optional[list] = None,
22
22
  return_fig: bool = False,
23
23
  ):
24
+ """
25
+ Display the image with the results
26
+ image_id: image id
27
+ display_fp: display false positive
28
+ display_fn: display false negative
29
+ display_tp: display true positive
30
+ display_gt: display ground truth
31
+ data_folder: data folder
32
+ categories: categories to display
33
+ return_fig: return the figure
34
+ """
24
35
  return display_image(
25
36
  self.cocoGt,
26
37
  self.cocoDt,
@@ -58,7 +69,13 @@ class PreviewResults(ExtraEval):
58
69
  )
59
70
 
60
71
  def _compute_confusion_matrix(self, y_true, y_pred, fp={}, fn={}):
61
- """Return classes*(classes + fp col + fn col)"""
72
+ """
73
+ Compute the confusion matrix
74
+ y_true: true labels
75
+ y_pred: predicted labels
76
+ fp: false positive
77
+ fn: false negative
78
+ """
62
79
  categories_real_ids = list(self.cocoGt.cats)
63
80
  categories_enum_ids = {
64
81
  category_id: _i
@@ -77,6 +94,7 @@ class PreviewResults(ExtraEval):
77
94
  return cm
78
95
 
79
96
  def compute_confusion_matrix(self):
97
+ """Compute the confusion matrix."""
80
98
  assert self.eval is not None, "Run first self.evaluate()"
81
99
 
82
100
  if self.useCats:
@@ -116,6 +134,12 @@ class PreviewResults(ExtraEval):
116
134
  def display_matrix(
117
135
  self, normalize=False, conf_matrix=None, return_fig: bool = False
118
136
  ):
137
+ """
138
+ Display the confusion matrix
139
+ normalize: normalize the matrix
140
+ conf_matrix: confusion matrix to display
141
+ return_fig: return the figure
142
+ """
119
143
  if conf_matrix is None:
120
144
  conf_matrix = self.compute_confusion_matrix()
121
145
 
@@ -27,7 +27,17 @@ def generate_ann_polygon(
27
27
  text: Optional[str] = None,
28
28
  legendgroup: Optional[str] = None,
29
29
  category_id_to_skeleton: Optional[dict] = None,
30
- ):
30
+ ) -> go.Scatter:
31
+ """Generate annotation polygon for plotly.
32
+
33
+ ann: annotation dictionary
34
+ color: color of the annotation
35
+ iouType: type of the annotation bbox or segm or keypoints
36
+ text: text to display
37
+ legendgroup: legend group to display
38
+ category_id_to_skeleton: dictionary of category id to skeleton
39
+
40
+ """
31
41
  all_x = []
32
42
  all_y = []
33
43
 
@@ -96,7 +106,21 @@ def display_image(
96
106
  data_folder: Optional[str] = None,
97
107
  categories: Optional[list] = None,
98
108
  return_fig: bool = False,
99
- ):
109
+ ) -> Optional[go.Figure]:
110
+ """
111
+ Display the image with the results
112
+ cocoGt: ground truth
113
+ cocoDt: detection
114
+ image_id: image id
115
+ iouType: type of the annotation bbox or segm or keypoints
116
+ display_fp: display false positive
117
+ display_fn: display false negative
118
+ display_tp: display true positive
119
+ display_gt: display ground truth
120
+ data_folder: data folder
121
+ categories: categories to display
122
+ return_fig: return the figure
123
+ """
100
124
  polygons = []
101
125
 
102
126
  image = cocoGt.imgs[image_id]
@@ -253,7 +277,14 @@ def display_matrix(
253
277
  labels: list,
254
278
  normalize: bool = False,
255
279
  return_fig: bool = False,
256
- ):
280
+ ) -> Optional[go.Figure]:
281
+ """
282
+ Display the confusion matrix
283
+ conf_matrix: confusion matrix
284
+ labels: labels of the matrix
285
+ normalize: normalize the matrix
286
+ return_fig: return the figure
287
+ """
257
288
  _labels = labels + ["fp", "fn"]
258
289
 
259
290
  if normalize:
@@ -316,6 +347,11 @@ def display_matrix(
316
347
 
317
348
 
318
349
  def plot_pre_rec(curves, return_fig: bool = False):
350
+ """
351
+ Plot the precision-recall curve
352
+ curves: list of curves to plot
353
+ return_fig: return the figure
354
+ """
319
355
  fig = go.Figure()
320
356
 
321
357
  for _curve in curves:
@@ -364,11 +400,16 @@ def plot_pre_rec(curves, return_fig: bool = False):
364
400
 
365
401
 
366
402
  def plot_f1_confidence(curves, return_fig: bool = False):
403
+ """
404
+ Plot the F1 confidence curve
405
+ curves: list of curves to plot
406
+ return_fig: return the figure
407
+ """
367
408
  fig = go.Figure()
368
409
  eps = 1e-16
369
410
  for _curve in curves:
370
411
  recall_list = _curve["recall_list"]
371
- precision_list = _curve["precision_list"]
412
+ precision_list = _curve["precision_list"][: len(recall_list)]
372
413
  scores = _curve["scores"]
373
414
  f1_curve = (
374
415
  2
@@ -414,3 +455,79 @@ def plot_f1_confidence(curves, return_fig: bool = False):
414
455
  return fig
415
456
 
416
457
  fig.show()
458
+
459
+
460
+ def plot_ced_metric(curves, normalize: bool = False, return_fig: bool = False):
461
+ fig = go.Figure()
462
+
463
+ if normalize:
464
+ fig.layout.yaxis.title = (
465
+ "The proportion of the sample to the total sample [%]"
466
+ )
467
+ _hovertemplate_y = "%{y:.2f}%<br>"
468
+ else:
469
+ fig.layout.yaxis.title = "Number of samples"
470
+ _hovertemplate_y = "n=%{y}<br>"
471
+
472
+ for ced_curve in curves:
473
+ if normalize:
474
+ y = (np.array(ced_curve["count"]) / ced_curve["total_count"]) * 100
475
+ else:
476
+ y = ced_curve["count"]
477
+
478
+ fig.layout.xaxis.title = "Mean squared error"
479
+
480
+ category_name = ced_curve["category"]["name"]
481
+ fig.add_trace(
482
+ go.Scatter(
483
+ x=ced_curve["mse"],
484
+ y=y,
485
+ name=f"CED Curve [{category_name}]",
486
+ hovertemplate=_hovertemplate_y
487
+ + "mse: %{x:.2f}<br><extra></extra>",
488
+ showlegend=True,
489
+ mode="lines",
490
+ )
491
+ )
492
+
493
+ fig.update_xaxes(showspikes=True)
494
+ fig.update_yaxes(showspikes=True)
495
+
496
+ updatemenus = [
497
+ dict(
498
+ type="dropdown",
499
+ direction="down",
500
+ y=1.1,
501
+ x=1,
502
+ buttons=list(
503
+ [
504
+ dict(
505
+ args=[{"xaxis.type": "linear"}],
506
+ label="Linear Scale",
507
+ method="relayout",
508
+ ),
509
+ dict(
510
+ args=[{"xaxis.type": "log"}],
511
+ label="Log Scale",
512
+ method="relayout",
513
+ ),
514
+ ]
515
+ ),
516
+ ),
517
+ ]
518
+
519
+ layout = {
520
+ "title": "Cumulative Error Distribution",
521
+ "autosize": True,
522
+ "height": 600,
523
+ "width": 1200,
524
+ # "xaxis_type" : "log",
525
+ "updatemenus": updatemenus,
526
+ }
527
+
528
+ fig.update_layout(layout)
529
+
530
+ if return_fig:
531
+ return fig
532
+
533
+ fig.show()
@@ -9,6 +9,8 @@ logger = logging.getLogger(__name__)
9
9
 
10
10
 
11
11
  class ExtraEval:
12
+ """Extra evaluation for coco dataset."""
13
+
12
14
  def __init__(
13
15
  self,
14
16
  cocoGt: COCO = None,
@@ -20,6 +22,16 @@ class ExtraEval:
20
22
  useCats: bool = False,
21
23
  kpt_oks_sigmas: list = None,
22
24
  ):
25
+ """
26
+ :param cocoGt (COCO):
27
+ :param cocoDt (COCO):
28
+ :param iouType (str): bbox, segm, keypoints
29
+ :param min_score (float):
30
+ :param iou_tresh (float):
31
+ :param recall_count (int):
32
+ :param useCats (bool):
33
+ :param kpt_oks_sigmas (list):
34
+ """
23
35
  self.iouType = iouType
24
36
  self.min_score = min_score
25
37
  self.iou_tresh = iou_tresh
@@ -53,11 +65,14 @@ class ExtraEval:
53
65
  )
54
66
  cocoEval.params.maxDets = [len(self.cocoGt.anns)]
55
67
 
56
- cocoEval.params.iouThrs = [self.iou_tresh]
57
- cocoEval.params.areaRng = [[0, 10000000000]]
58
68
  self.recThrs = np.linspace(0, 1, self.recall_count + 1, endpoint=True)
69
+
59
70
  cocoEval.params.recThrs = self.recThrs
60
71
 
72
+ if self.iouType != "keypoints":
73
+ cocoEval.params.iouThrs = [self.iou_tresh]
74
+
75
+ cocoEval.params.areaRng = [[0, 10000000000]]
61
76
  cocoEval.params.useCats = int(self.useCats) # Выключение labels
62
77
 
63
78
  self.cocoEval = cocoEval
@@ -68,6 +83,10 @@ class ExtraEval:
68
83
  self.eval = cocoEval.eval
69
84
 
70
85
  def drop_cocodt_by_score(self, min_score: float):
86
+ """
87
+ :param min_score:
88
+ :return:
89
+ """
71
90
  assert self.cocoDt is not None, "cocoDt is empty"
72
91
 
73
92
  if min_score > 0:
@@ -1,2 +1,2 @@
1
- __version__ = "1.5.4"
1
+ __version__ = "1.5.5"
2
2
  __author__ = "MiXaiLL76"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: faster-coco-eval
3
- Version: 1.5.4
3
+ Version: 1.5.5
4
4
  Summary: Faster interpretation of the original COCOEval
5
5
  Home-page: https://github.com/MiXaiLL76/faster_coco_eval
6
6
  Author: MiXaiLL76
@@ -53,7 +53,7 @@ For our use case with a test dataset of 5000 images from the coco val dataset.
53
53
  Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
54
54
 
55
55
  Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://github.com/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
56
- [colab_example.ipynb in google collab](https://drive.google.com/file/d/13raC0nRsil797ITJSv3FTRvUvfEK9SD-/view?usp=sharing)
56
+ [colab_example.ipynb in google collab](https://colab.research.google.com/drive/1qj392oIU8fmeyIFHCtxCrLA8PQQAMoIs)
57
57
  Tested with rtmdet model bbox + segm
58
58
 
59
59
  ### Summary for 5000 imgs
@@ -76,6 +76,10 @@ Code examples for using the library are available on the [Wiki](https://github.c
76
76
 
77
77
  Available via link [history.md](https://github.com/MiXaiLL76/faster_coco_eval/blob/main/history.md)
78
78
 
79
+ ## Star History
80
+
81
+ [![Star History Chart](https://api.star-history.com/svg?repos=MiXaiLL76/faster_coco_eval&type=Date)](https://star-history.com/#MiXaiLL76/faster_coco_eval&Date)
82
+
79
83
  ## License
80
84
 
81
85
  The original module was licensed with apache 2, I will continue with the same license.
@@ -1,63 +0,0 @@
1
- import logging
2
-
3
- from ..core import COCOeval_faster
4
- from .draw import plot_f1_confidence, plot_pre_rec
5
- from .extra import ExtraEval
6
-
7
- logger = logging.getLogger(__name__)
8
-
9
-
10
- class Curves(ExtraEval):
11
- def build_curve(self, label):
12
- assert self.eval is not None, "Run first self.evaluate()"
13
-
14
- curve = []
15
-
16
- if self.useCats:
17
- cat_ids = list(range(self.eval["precision"].shape[2]))
18
- else:
19
- cat_ids = [0]
20
-
21
- for category_id in cat_ids:
22
- _label = "[{}={}] ".format(label, category_id)
23
- if len(cat_ids) == 1:
24
- _label = ""
25
-
26
- precision_list = self.eval["precision"][
27
- :, :, category_id, :, :
28
- ].ravel()
29
- recall_list = self.recThrs
30
- scores = self.eval["scores"][:, :, category_id, :, :].ravel()
31
- auc = round(
32
- COCOeval_faster.calc_auc(recall_list, precision_list), 4
33
- )
34
-
35
- curve.append(
36
- dict(
37
- recall_list=recall_list,
38
- precision_list=precision_list,
39
- name="{}auc: {:.3f}".format(_label, auc),
40
- label=_label,
41
- scores=scores,
42
- auc=auc,
43
- category_id=category_id,
44
- )
45
- )
46
-
47
- return curve
48
-
49
- def plot_pre_rec(
50
- self, curves=None, label: str = "category_id", return_fig: bool = False
51
- ):
52
- if curves is None:
53
- curves = self.build_curve(label)
54
-
55
- return plot_pre_rec(curves, return_fig=return_fig)
56
-
57
- def plot_f1_confidence(
58
- self, curves=None, label: str = "category_id", return_fig: bool = False
59
- ):
60
- if curves is None:
61
- curves = self.build_curve(label)
62
-
63
- return plot_f1_confidence(curves, return_fig=return_fig)