faster-coco-eval 1.5.6__tar.gz → 1.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {faster_coco_eval-1.5.6/faster_coco_eval.egg-info → faster_coco_eval-1.6.0}/PKG-INFO +43 -6
  2. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/README.md +40 -3
  3. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/coco_eval/cocoeval.cpp +284 -20
  4. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/coco_eval/cocoeval.h +57 -1
  5. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/faster_eval_api.cpp +15 -1
  6. faster_coco_eval-1.6.0/csrc/mask_api/mask_api.cpp +132 -0
  7. faster_coco_eval-1.6.0/csrc/mask_api/src/mask.cpp +696 -0
  8. faster_coco_eval-1.6.0/csrc/mask_api/src/mask.h +105 -0
  9. faster_coco_eval-1.6.0/csrc/mask_api/src/rle.cpp +559 -0
  10. faster_coco_eval-1.6.0/faster_coco_eval/__init__.py +28 -0
  11. faster_coco_eval-1.6.0/faster_coco_eval/core/coco.py +698 -0
  12. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/core/cocoeval.py +367 -247
  13. faster_coco_eval-1.6.0/faster_coco_eval/core/faster_eval_api.py +277 -0
  14. faster_coco_eval-1.6.0/faster_coco_eval/core/mask.py +257 -0
  15. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/curves.py +49 -14
  16. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/display.py +49 -24
  17. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/draw.py +109 -48
  18. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/extra.py +18 -0
  19. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/utils.py +20 -9
  20. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/version.py +1 -1
  21. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0/faster_coco_eval.egg-info}/PKG-INFO +43 -6
  22. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/SOURCES.txt +11 -4
  23. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/requires.txt +1 -0
  24. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/pyproject.toml +2 -2
  25. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/requirements/tests.txt +1 -0
  26. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/setup.py +21 -27
  27. faster_coco_eval-1.6.0/tests/test_basic.py +235 -0
  28. faster_coco_eval-1.6.0/tests/test_boundary.py +142 -0
  29. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/tests/test_coco_metric.py +58 -0
  30. faster_coco_eval-1.6.0/tests/test_init_pycocotools.py +34 -0
  31. faster_coco_eval-1.6.0/tests/test_keypoints.py +70 -0
  32. faster_coco_eval-1.6.0/tests/test_lvis_metric.py +207 -0
  33. faster_coco_eval-1.6.0/tests/test_mask_api.py +324 -0
  34. faster_coco_eval-1.6.0/tests/test_torchmetrics.py +103 -0
  35. faster_coco_eval-1.5.6/csrc/mask/common/maskApi.c +0 -267
  36. faster_coco_eval-1.5.6/csrc/mask/common/maskApi.h +0 -60
  37. faster_coco_eval-1.5.6/csrc/mask/pycocotools/_mask.pyx +0 -312
  38. faster_coco_eval-1.5.6/faster_coco_eval/__init__.py +0 -4
  39. faster_coco_eval-1.5.6/faster_coco_eval/core/coco.py +0 -548
  40. faster_coco_eval-1.5.6/faster_coco_eval/core/faster_eval_api.py +0 -348
  41. faster_coco_eval-1.5.6/faster_coco_eval/core/mask.py +0 -111
  42. faster_coco_eval-1.5.6/tests/test_lvis_metric.py +0 -62
  43. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/LICENSE +0 -0
  44. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/MANIFEST.in +0 -0
  45. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/core/__init__.py +0 -0
  46. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/__init__.py +0 -0
  47. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/dependency_links.txt +0 -0
  48. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/top_level.txt +0 -0
  49. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/requirements/extra.txt +0 -0
  50. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/requirements/runtime.txt +0 -0
  51. {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: faster-coco-eval
3
- Version: 1.5.6
3
+ Version: 1.6.0
4
4
  Summary: Faster interpretation of the original COCOEval
5
5
  Home-page: https://github.com/MiXaiLL76/faster_coco_eval
6
6
  Author: MiXaiLL76
@@ -9,13 +9,12 @@ Classifier: Development Status :: 4 - Beta
9
9
  Classifier: License :: OSI Approved :: Apache Software License
10
10
  Classifier: Operating System :: OS Independent
11
11
  Classifier: Programming Language :: Python :: 3
12
- Classifier: Programming Language :: Python :: 3.6
13
12
  Classifier: Programming Language :: Python :: 3.7
14
13
  Classifier: Programming Language :: Python :: 3.8
15
14
  Classifier: Programming Language :: Python :: 3.9
16
15
  Classifier: Programming Language :: Python :: 3.10
17
16
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
- Requires-Python: >=3.6
17
+ Requires-Python: >=3.7
19
18
  Description-Content-Type: text/markdown
20
19
  License-File: LICENSE
21
20
  Requires-Dist: numpy<2
@@ -36,6 +35,7 @@ Requires-Dist: Pillow; extra == "tests"
36
35
  Requires-Dist: numpy<2; extra == "tests"
37
36
  Requires-Dist: pycocotools; extra == "tests"
38
37
  Requires-Dist: pytest; extra == "tests"
38
+ Requires-Dist: pytest-cov; extra == "tests"
39
39
  Requires-Dist: parameterized; extra == "tests"
40
40
 
41
41
  # Faster-COCO-Eval
@@ -55,6 +55,43 @@ https://pypi.org/project/faster-coco-eval/)
55
55
  I often use this project, but I saw it abandoned and without a public repository on github.
56
56
  Also, part of the project remained unfinished for a long time. I implemented some of the author's ideas and decided to make the results publicly available.
57
57
 
58
+ ## Install
59
+
60
+ ### Basic implementation identical to pycocotools
61
+
62
+ ```bash
63
+ pip install faster-coco-eval
64
+ ```
65
+
66
+ ### Additional visualization options
67
+
68
+ > Only 1 additional package needed opencv-python-headless
69
+
70
+ ```bash
71
+ pip install faster-coco-eval[extra]
72
+ ```
73
+
74
+ ### Basic usage
75
+
76
+ ```py
77
+ import faster_coco_eval
78
+
79
+ # Replace pycocotools with faster_coco_eval
80
+ faster_coco_eval.init_as_pycocotools()
81
+
82
+ from pycocotools.coco import COCO
83
+ from pycocotools.cocoeval import COCOeval
84
+
85
+ anno = COCO(str(anno_json)) # init annotations api
86
+ pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path)
87
+
88
+ val = COCOeval(anno, pred, "bbox")
89
+ val.evaluate()
90
+ val.accumulate()
91
+ val.summarize()
92
+
93
+ ```
94
+
58
95
  ## Faster-COCO-Eval base
59
96
 
60
97
  This package wraps a facebook C++ implementation of COCO-eval operations found in the
@@ -67,9 +104,9 @@ for coco's AP metrics, especially when dealing with a high number of instances i
67
104
  For our use case with a test dataset of 5000 images from the coco val dataset.
68
105
  Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
69
106
 
70
- Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
71
- [colab_example.ipynb in google collab](https://colab.research.google.com/drive/1qj392oIU8fmeyIFHCtxCrLA8PQQAMoIs)
72
- Tested with rtmdet model bbox + segm
107
+ Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison)
108
+ - [mmdet example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
109
+ - [ultralytics example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/ultralytics/colab_example.ipynb)
73
110
 
74
111
  ### Summary for 5000 imgs
75
112
 
@@ -15,6 +15,43 @@ https://pypi.org/project/faster-coco-eval/)
15
15
  I often use this project, but I saw it abandoned and without a public repository on github.
16
16
  Also, part of the project remained unfinished for a long time. I implemented some of the author's ideas and decided to make the results publicly available.
17
17
 
18
+ ## Install
19
+
20
+ ### Basic implementation identical to pycocotools
21
+
22
+ ```bash
23
+ pip install faster-coco-eval
24
+ ```
25
+
26
+ ### Additional visualization options
27
+
28
+ > Only 1 additional package needed opencv-python-headless
29
+
30
+ ```bash
31
+ pip install faster-coco-eval[extra]
32
+ ```
33
+
34
+ ### Basic usage
35
+
36
+ ```py
37
+ import faster_coco_eval
38
+
39
+ # Replace pycocotools with faster_coco_eval
40
+ faster_coco_eval.init_as_pycocotools()
41
+
42
+ from pycocotools.coco import COCO
43
+ from pycocotools.cocoeval import COCOeval
44
+
45
+ anno = COCO(str(anno_json)) # init annotations api
46
+ pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path)
47
+
48
+ val = COCOeval(anno, pred, "bbox")
49
+ val.evaluate()
50
+ val.accumulate()
51
+ val.summarize()
52
+
53
+ ```
54
+
18
55
  ## Faster-COCO-Eval base
19
56
 
20
57
  This package wraps a facebook C++ implementation of COCO-eval operations found in the
@@ -27,9 +64,9 @@ for coco's AP metrics, especially when dealing with a high number of instances i
27
64
  For our use case with a test dataset of 5000 images from the coco val dataset.
28
65
  Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
29
66
 
30
- Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
31
- [colab_example.ipynb in google collab](https://colab.research.google.com/drive/1qj392oIU8fmeyIFHCtxCrLA8PQQAMoIs)
32
- Tested with rtmdet model bbox + segm
67
+ Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison)
68
+ - [mmdet example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
69
+ - [ultralytics example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/ultralytics/colab_example.ipynb)
33
70
 
34
71
  ### Summary for 5000 imgs
35
72
 
@@ -11,6 +11,20 @@ namespace coco_eval
11
11
 
12
12
  namespace COCOeval
13
13
  {
14
+ template <typename T>
15
+ int v_index(const std::vector<T> &v, const T &key)
16
+ {
17
+ auto itr = std::find(v.begin(), v.end(), key);
18
+
19
+ if (itr != v.cend())
20
+ {
21
+ return std::distance(v.begin(), itr);
22
+ }
23
+ else
24
+ {
25
+ return -1;
26
+ }
27
+ }
14
28
 
15
29
  // Sort detections from highest score to lowest, such that
16
30
  // detection_instances[detection_sorted_indices[t]] >=
@@ -44,7 +58,7 @@ namespace coco_eval
44
58
  ignores->reserve(ground_truth_instances.size());
45
59
  for (auto o : ground_truth_instances)
46
60
  {
47
- ignores->push_back(
61
+ ignores->emplace_back(
48
62
  o.ignore || o.area < area_range[0] || o.area > area_range[1]);
49
63
  }
50
64
 
@@ -84,9 +98,6 @@ namespace coco_eval
84
98
  std::vector<int64_t> &ground_truth_matches = results->ground_truth_matches;
85
99
  ground_truth_matches.resize(num_iou_thresholds * num_ground_truth, 0);
86
100
 
87
- std::vector<int64_t> &ground_truth_orig_id = results->ground_truth_orig_id;
88
- ground_truth_orig_id.resize(num_iou_thresholds * num_ground_truth, -1);
89
-
90
101
  std::vector<int64_t> &detection_matches = results->detection_matches;
91
102
 
92
103
  std::vector<bool> &detection_ignores = results->detection_ignores;
@@ -141,7 +152,12 @@ namespace coco_eval
141
152
  ground_truth_instances[ground_truth_sorted_indices[match]].id;
142
153
  ground_truth_matches[t * num_ground_truth + match] =
143
154
  detection_instances[detection_sorted_indices[d]].id;
144
- ground_truth_orig_id[t * num_ground_truth + match] = (long)ground_truth_instances[ground_truth_sorted_indices[match]].id;
155
+
156
+ results->matched_annotations.push_back(
157
+ MatchedAnnotation(
158
+ ground_truth_matches[t * num_ground_truth + match], // DT_ID
159
+ detection_matches[t * num_detections + d], // GT_ID
160
+ best_iou));
145
161
  }
146
162
 
147
163
  // set unmatched detections outside of area range to ignore
@@ -280,9 +296,9 @@ namespace coco_eval
280
296
  d < (int)evaluation.detection_scores.size() && d < max_detections;
281
297
  ++d)
282
298
  { // detected instances
283
- evaluation_indices->push_back(evaluation_index + i);
284
- image_detection_indices->push_back(d);
285
- detection_scores->push_back(evaluation.detection_scores[d]);
299
+ evaluation_indices->emplace_back(evaluation_index + i);
300
+ image_detection_indices->emplace_back(d);
301
+ detection_scores->emplace_back(evaluation.detection_scores[d]);
286
302
  }
287
303
  for (auto ground_truth_ignore : evaluation.ground_truth_ignores)
288
304
  {
@@ -373,13 +389,13 @@ namespace coco_eval
373
389
 
374
390
  const double recall =
375
391
  static_cast<double>(true_positives_sum) / num_valid_ground_truth;
376
- recalls->push_back(recall);
392
+ recalls->emplace_back(recall);
377
393
  const int64_t num_valid_detections =
378
394
  true_positives_sum + false_positives_sum;
379
395
  const double precision = num_valid_detections > 0
380
396
  ? static_cast<double>(true_positives_sum) / num_valid_detections
381
397
  : 0.0;
382
- precisions->push_back(precision);
398
+ precisions->emplace_back(precision);
383
399
  }
384
400
 
385
401
  (*recalls_out)[recalls_out_index] = !recalls->empty() ? recalls->back() : 0;
@@ -544,14 +560,26 @@ namespace coco_eval
544
560
 
545
561
  int evaluations_size = static_cast<int>(evaluations.size());
546
562
 
547
- std::vector<int64_t> out_detection_matches = {};
548
- std::vector<int64_t> out_ground_truth_matches = {};
549
- std::vector<int64_t> out_ground_truth_orig_id = {};
563
+ std::unordered_map<std::string, double> matched;
564
+
550
565
  for (auto eval : evaluations)
551
566
  {
552
- out_detection_matches.insert(out_detection_matches.end(), eval.detection_matches.begin(), eval.detection_matches.end());
553
- out_ground_truth_matches.insert(out_ground_truth_matches.end(), eval.ground_truth_matches.begin(), eval.ground_truth_matches.end());
554
- out_ground_truth_orig_id.insert(out_ground_truth_orig_id.end(), eval.ground_truth_orig_id.begin(), eval.ground_truth_orig_id.end());
567
+ for (auto matched_annotation : eval.matched_annotations)
568
+ {
569
+ std::string key = std::to_string(matched_annotation.dt_id) + "_" + std::to_string(matched_annotation.gt_id);
570
+
571
+ if (matched.find(key) != matched.end())
572
+ {
573
+ if (matched[key] < matched_annotation.iou)
574
+ {
575
+ matched[key] = matched_annotation.iou;
576
+ }
577
+ }
578
+ else
579
+ {
580
+ matched[key] = matched_annotation.iou;
581
+ }
582
+ }
555
583
  }
556
584
 
557
585
  std::vector<int64_t> counts = {
@@ -573,16 +601,14 @@ namespace coco_eval
573
601
  "counts"_a = counts,
574
602
  "date"_a = py::str(buffer),
575
603
 
604
+ "matched"_a = matched,
605
+
576
606
  // precision and scores are num_iou_thresholds X num_recall_thresholds X num_categories X num_area_ranges X num_max_detections
577
607
  "precision"_a = py::array(precisions_out.size(), precisions_out.data()).reshape(counts),
578
608
  "scores"_a = py::array(scores_out.size(), scores_out.data()).reshape(counts),
579
609
 
580
610
  // recall is num_iou_thresholds X num_categories X num_area_ranges X num_max_detections
581
611
  "recall"_a = py::array(recalls_out.size(), recalls_out.data()).reshape(recall_counts),
582
-
583
- "detection_matches"_a = py::array(out_detection_matches.size(), out_detection_matches.data()).reshape(matches_shape),
584
- "ground_truth_matches"_a = py::array(out_ground_truth_matches.size(), out_ground_truth_matches.data()).reshape(matches_shape),
585
- "ground_truth_orig_id"_a = py::array(out_ground_truth_orig_id.size(), out_ground_truth_orig_id.data()).reshape(matches_shape),
586
612
  "evaluations_size"_a = evaluations_size);
587
613
  }
588
614
 
@@ -601,6 +627,244 @@ namespace coco_eval
601
627
  std::vector<ImageEvaluation> result = EvaluateImages(area_ranges, max_detections.back(), iou_thresholds, image_category_ious, image_category_ground_truth_instances, image_category_detection_instances);
602
628
  return Accumulate(params, result);
603
629
  }
630
+
631
+ void Dataset::append(int64_t img_id, int64_t cat_id, py::dict ann)
632
+ {
633
+ this->data[{img_id, cat_id}].emplace_back(ann);
634
+ }
635
+ std::vector<py::dict> Dataset::get(const int64_t &img_id, const int64_t &cat_id)
636
+ {
637
+ std::pair<int64_t, int64_t> key(img_id, cat_id);
638
+
639
+ if (this->data.find(key) != this->data.end())
640
+ {
641
+ return this->data[key];
642
+ }
643
+ else
644
+ {
645
+ return std::vector<py::dict>();
646
+ }
647
+ }
648
+
649
+ InstanceAnnotation parseInstanceAnnotation(const py::dict &ann)
650
+ {
651
+ uint64_t id = 0;
652
+ double score = 0.;
653
+ double area = 0.;
654
+ bool is_crowd = false;
655
+ bool ignore = false;
656
+ bool lvis_mark = false;
657
+
658
+ for (auto it : ann)
659
+ {
660
+ std::string key = it.first.cast<std::string>();
661
+
662
+ if (key == "id")
663
+ {
664
+ id = it.second.cast<uint64_t>();
665
+ }
666
+ else if (key == "score")
667
+ {
668
+ score = it.second.cast<double>();
669
+ }
670
+ else if (key == "area")
671
+ {
672
+ area = it.second.cast<double>();
673
+ }
674
+ else if ((key == "is_crowd") || key == "iscrowd")
675
+ {
676
+ is_crowd = it.second.cast<bool>();
677
+ }
678
+ else if (key == "ignore")
679
+ {
680
+ ignore = it.second.cast<bool>();
681
+ }
682
+ else if (key == "lvis_mark")
683
+ {
684
+ lvis_mark = it.second.cast<bool>();
685
+ }
686
+ }
687
+ return InstanceAnnotation(id, score, area, is_crowd, ignore, lvis_mark);
688
+ }
689
+
690
+ std::vector<InstanceAnnotation> Dataset::get_cpp_annotations(
691
+ const int64_t &img_id, const int64_t &cat_id)
692
+ {
693
+ std::vector<py::dict> anns = get(img_id, cat_id);
694
+ std::vector<InstanceAnnotation> result;
695
+ for (size_t i = 0; i < anns.size(); i++)
696
+ {
697
+ result.push_back(parseInstanceAnnotation(anns[i]));
698
+ }
699
+ return result;
700
+ }
701
+
702
+ std::vector<std::vector<std::vector<InstanceAnnotation>>> Dataset::get_cpp_instances(
703
+ const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats)
704
+ {
705
+ std::vector<std::vector<std::vector<InstanceAnnotation>>> result;
706
+ for (size_t i = 0; i < img_ids.size(); i++)
707
+ {
708
+ int64_t img_id = img_ids[i];
709
+ result.push_back(std::vector<std::vector<InstanceAnnotation>>());
710
+ if (!useCats)
711
+ {
712
+ result[i].push_back(std::vector<InstanceAnnotation>());
713
+ }
714
+
715
+ for (size_t j = 0; j < cat_ids.size(); j++)
716
+ {
717
+ int64_t cat_id = cat_ids[j];
718
+
719
+ std::vector<InstanceAnnotation> anns = this->get_cpp_annotations(img_id, cat_id);
720
+
721
+ if (useCats)
722
+ {
723
+ result[i].emplace_back(anns);
724
+ }
725
+ else
726
+ {
727
+ result[i][0].insert(result[i][0].end(), anns.begin(), anns.end());
728
+ }
729
+ }
730
+ }
731
+ return result;
732
+ }
733
+
734
+ std::vector<std::vector<std::vector<py::dict>>> Dataset::get_instances(
735
+ const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats)
736
+ {
737
+ std::vector<std::vector<std::vector<py::dict>>> result;
738
+ for (size_t i = 0; i < img_ids.size(); i++)
739
+ {
740
+ int64_t img_id = img_ids[i];
741
+ result.push_back(std::vector<std::vector<py::dict>>());
742
+ if (!useCats)
743
+ {
744
+ result[i].push_back(std::vector<py::dict>());
745
+ }
746
+
747
+ for (size_t j = 0; j < cat_ids.size(); j++)
748
+ {
749
+ int64_t cat_id = cat_ids[j];
750
+
751
+ std::vector<py::dict> anns = this->get(img_id, cat_id);
752
+
753
+ if (useCats)
754
+ {
755
+ result[i].emplace_back(anns);
756
+ }
757
+ else
758
+ {
759
+ result[i][0].insert(result[i][0].end(), anns.begin(), anns.end());
760
+ }
761
+ }
762
+ }
763
+ return result;
764
+ }
765
+
766
+ long double calc_auc(const std::vector<long double> &recall_list, const std::vector<long double> &precision_list)
767
+ {
768
+ std::vector<long double> mpre = precision_list;
769
+
770
+ for (size_t i = mpre.size() - 1; i > 0; i--)
771
+ {
772
+ mpre[i - 1] = std::max(mpre[i - 1], mpre[i]);
773
+ }
774
+
775
+ long double result = 0;
776
+
777
+ for (size_t i = 1; i < recall_list.size(); i++)
778
+ {
779
+ if (recall_list[i - 1] != recall_list[i])
780
+ {
781
+ result += (recall_list[i] - recall_list[i - 1]) * mpre[i];
782
+ }
783
+ }
784
+
785
+ return result;
786
+ }
787
+
788
+ long double _summarize(const int &ap, const double &iouThr, const std::string &areaRng, const int &maxDet, const std::vector<int> &catIds, const py::object &params, const std::vector<size_t> &counts, const py::object &nums_array)
789
+ {
790
+ std::vector<std::string> areaRngLbl = list_to_vec<std::string>(params.attr("areaRngLbl"));
791
+ std::vector<int> maxDets = list_to_vec<int>(params.attr("maxDets"));
792
+ std::vector<double> iouThrs = list_to_vec<double>(params.attr("iouThrs"));
793
+
794
+ std::vector<int> _catIds;
795
+
796
+ int iou_ind = v_index(iouThrs, iouThr);
797
+ int aind = v_index(areaRngLbl, areaRng);
798
+ int mind = v_index(maxDets, maxDet);
799
+
800
+ if (catIds.size() == 0)
801
+ {
802
+ for (size_t category = 0; category < counts[2]; category++)
803
+ {
804
+ _catIds.emplace_back(category);
805
+ }
806
+ }
807
+ else
808
+ {
809
+ _catIds = catIds;
810
+ }
811
+
812
+ std::vector<long double> result;
813
+
814
+ if (ap == 1)
815
+ {
816
+ // # dimension of precision: [TxRxKxAxM]
817
+ std::vector<std::vector<std::vector<std::vector<std::vector<double>>>>> precision = nums_array.cast<std::vector<std::vector<std::vector<std::vector<std::vector<double>>>>>>();
818
+ if (iou_ind != -1)
819
+ {
820
+ precision = {precision[iou_ind]};
821
+ }
822
+ for (size_t iou_threshold = 0; iou_threshold < precision.size(); iou_threshold++)
823
+ {
824
+ for (size_t recall_threshold = 0; recall_threshold < counts[1]; recall_threshold++)
825
+ {
826
+ for (const auto &category : _catIds)
827
+ {
828
+ long double pre = (long double)precision[iou_threshold][recall_threshold][category][aind][mind];
829
+ if (pre != -1)
830
+ {
831
+ result.emplace_back(pre);
832
+ }
833
+ }
834
+ }
835
+ }
836
+ }
837
+ else
838
+ {
839
+ // # dimension of recall: [TxKxAxM]
840
+ std::vector<std::vector<std::vector<std::vector<double>>>> recall = nums_array.cast<std::vector<std::vector<std::vector<std::vector<double>>>>>();
841
+ if (iou_ind != -1)
842
+ {
843
+ recall = {recall[iou_ind]};
844
+ }
845
+ for (size_t iou_threshold = 0; iou_threshold < recall.size(); iou_threshold++)
846
+ {
847
+ for (const auto &category : catIds)
848
+ {
849
+ long double rec = (long double)recall[iou_threshold][category][aind][mind];
850
+ if (rec != -1)
851
+ {
852
+ result.emplace_back(rec);
853
+ }
854
+ }
855
+ }
856
+ }
857
+
858
+ if (result.size() > 0)
859
+ {
860
+ return std::accumulate(result.begin(), result.end(), 0.0) / result.size();
861
+ }
862
+ else
863
+ {
864
+ return -1;
865
+ }
866
+ }
867
+
604
868
  } // namespace COCOeval
605
869
 
606
870
  } // namespace coco_eval
@@ -34,6 +34,18 @@ namespace coco_eval
34
34
  bool lvis_mark = false;
35
35
  };
36
36
 
37
+ // Stores the match between a detected instance and a ground truth instance
38
+ struct MatchedAnnotation
39
+ {
40
+ MatchedAnnotation(
41
+ uint64_t dt_id,
42
+ uint64_t gt_id,
43
+ double iou) : dt_id{dt_id}, gt_id{gt_id}, iou{iou} {}
44
+ uint64_t dt_id;
45
+ uint64_t gt_id;
46
+ double iou;
47
+ };
48
+
37
49
  // Stores intermediate results for evaluating detection results for a single
38
50
  // image that has D detected instances and G ground truth instances. This stores
39
51
  // matches between detected and ground truth instances
@@ -44,7 +56,6 @@ namespace coco_eval
44
56
  std::vector<int64_t> detection_matches;
45
57
 
46
58
  std::vector<int64_t> ground_truth_matches;
47
- std::vector<int64_t> ground_truth_orig_id;
48
59
  // The detection score of each of the D detected instances
49
60
  std::vector<double> detection_scores;
50
61
 
@@ -55,6 +66,49 @@ namespace coco_eval
55
66
  // Marks whether or not each of D instances was ignored from evaluation (e.g.,
56
67
  // because it's outside aRng)
57
68
  std::vector<bool> detection_ignores;
69
+
70
+ std::vector<MatchedAnnotation> matched_annotations;
71
+ };
72
+
73
+ template <class T>
74
+ inline void hash_combine(std::size_t &seed, const T &v)
75
+ {
76
+ std::hash<T> hasher;
77
+ seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
78
+ }
79
+
80
+ struct hash_pair
81
+ {
82
+ std::size_t operator()(const std::pair<int64_t, int64_t> &p) const
83
+ {
84
+ std::size_t h = 0;
85
+ hash_combine(h, p.first);
86
+ hash_combine(h, p.second);
87
+ return h;
88
+ }
89
+ };
90
+
91
+ class Dataset
92
+ {
93
+ public:
94
+ Dataset()
95
+ {
96
+ this->data.reserve(8192); // RESERVING SPACE BEFOREHAND
97
+ // this->data.max_load_factor(0.25); // DECREASING MAX_LOAD_FACTOR
98
+ }
99
+ void append(int64_t img_id, int64_t cat_id, py::dict ann);
100
+ std::vector<py::dict> get(const int64_t &img_id, const int64_t &cat_id);
101
+ std::vector<InstanceAnnotation> get_cpp_annotations(
102
+ const int64_t &img_id, const int64_t &cat_id);
103
+
104
+ std::vector<std::vector<std::vector<InstanceAnnotation>>> get_cpp_instances(
105
+ const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats);
106
+
107
+ std::vector<std::vector<std::vector<py::dict>>> get_instances(
108
+ const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats);
109
+
110
+ private:
111
+ std::unordered_map<std::pair<int64_t, int64_t>, std::vector<py::dict>, hash_pair> data;
58
112
  };
59
113
 
60
114
  template <class T>
@@ -100,5 +154,7 @@ namespace coco_eval
100
154
  const ImageCategoryInstances<InstanceAnnotation> &
101
155
  image_category_detection_instances);
102
156
 
157
+ long double calc_auc(const std::vector<long double> &recall_list, const std::vector<long double> &precision_list);
158
+ long double _summarize(const int &ap, const double &iouThr, const std::string &areaRng, const int &maxDet, const std::vector<int> &catIds, const py::object &params, const std::vector<size_t> &counts, const py::object &nums_array);
103
159
  } // namespace COCOeval
104
160
  } // namespace coco_eval
@@ -49,8 +49,22 @@ namespace coco_eval
49
49
  m.def("COCOevalAccumulate", &COCOeval::Accumulate, "COCOeval::Accumulate");
50
50
  m.def("COCOevalEvaluateImages", &COCOeval::EvaluateImages, "COCOeval::EvaluateImages");
51
51
  m.def("COCOevalEvaluateAccumulate", &COCOeval::EvaluateAccumulate, "COCOeval::EvaluateAccumulate");
52
- pybind11::class_<COCOeval::InstanceAnnotation>(m, "InstanceAnnotation").def(pybind11::init<uint64_t, double, double, bool, bool, bool>());
52
+
53
+ // slow!
54
+ m.def("_summarize", &COCOeval::_summarize, "COCOeval::_summarize");
55
+ m.def("calc_auc", &COCOeval::calc_auc, "COCOeval::calc_auc");
56
+
57
+ pybind11::class_<COCOeval::InstanceAnnotation>(m, "InstanceAnnotation")
58
+ .def(pybind11::init<uint64_t, double, double, bool, bool, bool>());
59
+
53
60
  pybind11::class_<COCOeval::ImageEvaluation>(m, "ImageEvaluation").def(pybind11::init<>());
61
+
62
+ pybind11::class_<COCOeval::Dataset>(m, "Dataset").def(pybind11::init<>())
63
+ .def("append", &COCOeval::Dataset::append)
64
+ .def("get", &COCOeval::Dataset::get)
65
+ .def("get_instances", &COCOeval::Dataset::get_instances)
66
+ .def("get_cpp_annotations", &COCOeval::Dataset::get_cpp_annotations)
67
+ .def("get_cpp_instances", &COCOeval::Dataset::get_cpp_instances);
54
68
 
55
69
  #ifdef VERSION_INFO
56
70
  m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO);