faster-coco-eval 1.5.6__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {faster_coco_eval-1.5.6/faster_coco_eval.egg-info → faster_coco_eval-1.6.0}/PKG-INFO +43 -6
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/README.md +40 -3
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/coco_eval/cocoeval.cpp +284 -20
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/coco_eval/cocoeval.h +57 -1
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/faster_eval_api.cpp +15 -1
- faster_coco_eval-1.6.0/csrc/mask_api/mask_api.cpp +132 -0
- faster_coco_eval-1.6.0/csrc/mask_api/src/mask.cpp +696 -0
- faster_coco_eval-1.6.0/csrc/mask_api/src/mask.h +105 -0
- faster_coco_eval-1.6.0/csrc/mask_api/src/rle.cpp +559 -0
- faster_coco_eval-1.6.0/faster_coco_eval/__init__.py +28 -0
- faster_coco_eval-1.6.0/faster_coco_eval/core/coco.py +698 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/core/cocoeval.py +367 -247
- faster_coco_eval-1.6.0/faster_coco_eval/core/faster_eval_api.py +277 -0
- faster_coco_eval-1.6.0/faster_coco_eval/core/mask.py +257 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/curves.py +49 -14
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/display.py +49 -24
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/draw.py +109 -48
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/extra.py +18 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/utils.py +20 -9
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/version.py +1 -1
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0/faster_coco_eval.egg-info}/PKG-INFO +43 -6
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/SOURCES.txt +11 -4
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/requires.txt +1 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/pyproject.toml +2 -2
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/requirements/tests.txt +1 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/setup.py +21 -27
- faster_coco_eval-1.6.0/tests/test_basic.py +235 -0
- faster_coco_eval-1.6.0/tests/test_boundary.py +142 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/tests/test_coco_metric.py +58 -0
- faster_coco_eval-1.6.0/tests/test_init_pycocotools.py +34 -0
- faster_coco_eval-1.6.0/tests/test_keypoints.py +70 -0
- faster_coco_eval-1.6.0/tests/test_lvis_metric.py +207 -0
- faster_coco_eval-1.6.0/tests/test_mask_api.py +324 -0
- faster_coco_eval-1.6.0/tests/test_torchmetrics.py +103 -0
- faster_coco_eval-1.5.6/csrc/mask/common/maskApi.c +0 -267
- faster_coco_eval-1.5.6/csrc/mask/common/maskApi.h +0 -60
- faster_coco_eval-1.5.6/csrc/mask/pycocotools/_mask.pyx +0 -312
- faster_coco_eval-1.5.6/faster_coco_eval/__init__.py +0 -4
- faster_coco_eval-1.5.6/faster_coco_eval/core/coco.py +0 -548
- faster_coco_eval-1.5.6/faster_coco_eval/core/faster_eval_api.py +0 -348
- faster_coco_eval-1.5.6/faster_coco_eval/core/mask.py +0 -111
- faster_coco_eval-1.5.6/tests/test_lvis_metric.py +0 -62
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/LICENSE +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/MANIFEST.in +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/core/__init__.py +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval/extra/__init__.py +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/dependency_links.txt +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/faster_coco_eval.egg-info/top_level.txt +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/requirements/extra.txt +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/requirements/runtime.txt +0 -0
- {faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: faster-coco-eval
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Summary: Faster interpretation of the original COCOEval
|
|
5
5
|
Home-page: https://github.com/MiXaiLL76/faster_coco_eval
|
|
6
6
|
Author: MiXaiLL76
|
|
@@ -9,13 +9,12 @@ Classifier: Development Status :: 4 - Beta
|
|
|
9
9
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.7
|
|
14
13
|
Classifier: Programming Language :: Python :: 3.8
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.9
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.10
|
|
17
16
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.7
|
|
19
18
|
Description-Content-Type: text/markdown
|
|
20
19
|
License-File: LICENSE
|
|
21
20
|
Requires-Dist: numpy<2
|
|
@@ -36,6 +35,7 @@ Requires-Dist: Pillow; extra == "tests"
|
|
|
36
35
|
Requires-Dist: numpy<2; extra == "tests"
|
|
37
36
|
Requires-Dist: pycocotools; extra == "tests"
|
|
38
37
|
Requires-Dist: pytest; extra == "tests"
|
|
38
|
+
Requires-Dist: pytest-cov; extra == "tests"
|
|
39
39
|
Requires-Dist: parameterized; extra == "tests"
|
|
40
40
|
|
|
41
41
|
# Faster-COCO-Eval
|
|
@@ -55,6 +55,43 @@ https://pypi.org/project/faster-coco-eval/)
|
|
|
55
55
|
I often use this project, but I saw it abandoned and without a public repository on github.
|
|
56
56
|
Also, part of the project remained unfinished for a long time. I implemented some of the author's ideas and decided to make the results publicly available.
|
|
57
57
|
|
|
58
|
+
## Install
|
|
59
|
+
|
|
60
|
+
### Basic implementation identical to pycocotools
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install faster-coco-eval
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### Additional visualization options
|
|
67
|
+
|
|
68
|
+
> Only 1 additional package needed opencv-python-headless
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install faster-coco-eval[extra]
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Basic usage
|
|
75
|
+
|
|
76
|
+
```py
|
|
77
|
+
import faster_coco_eval
|
|
78
|
+
|
|
79
|
+
# Replace pycocotools with faster_coco_eval
|
|
80
|
+
faster_coco_eval.init_as_pycocotools()
|
|
81
|
+
|
|
82
|
+
from pycocotools.coco import COCO
|
|
83
|
+
from pycocotools.cocoeval import COCOeval
|
|
84
|
+
|
|
85
|
+
anno = COCO(str(anno_json)) # init annotations api
|
|
86
|
+
pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path)
|
|
87
|
+
|
|
88
|
+
val = COCOeval(anno, pred, "bbox")
|
|
89
|
+
val.evaluate()
|
|
90
|
+
val.accumulate()
|
|
91
|
+
val.summarize()
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
|
|
58
95
|
## Faster-COCO-Eval base
|
|
59
96
|
|
|
60
97
|
This package wraps a facebook C++ implementation of COCO-eval operations found in the
|
|
@@ -67,9 +104,9 @@ for coco's AP metrics, especially when dealing with a high number of instances i
|
|
|
67
104
|
For our use case with a test dataset of 5000 images from the coco val dataset.
|
|
68
105
|
Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
|
|
69
106
|
|
|
70
|
-
Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison
|
|
71
|
-
[
|
|
72
|
-
|
|
107
|
+
Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison)
|
|
108
|
+
- [mmdet example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
|
|
109
|
+
- [ultralytics example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/ultralytics/colab_example.ipynb)
|
|
73
110
|
|
|
74
111
|
### Summary for 5000 imgs
|
|
75
112
|
|
|
@@ -15,6 +15,43 @@ https://pypi.org/project/faster-coco-eval/)
|
|
|
15
15
|
I often use this project, but I saw it abandoned and without a public repository on github.
|
|
16
16
|
Also, part of the project remained unfinished for a long time. I implemented some of the author's ideas and decided to make the results publicly available.
|
|
17
17
|
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
### Basic implementation identical to pycocotools
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install faster-coco-eval
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
### Additional visualization options
|
|
27
|
+
|
|
28
|
+
> Only 1 additional package needed opencv-python-headless
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install faster-coco-eval[extra]
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Basic usage
|
|
35
|
+
|
|
36
|
+
```py
|
|
37
|
+
import faster_coco_eval
|
|
38
|
+
|
|
39
|
+
# Replace pycocotools with faster_coco_eval
|
|
40
|
+
faster_coco_eval.init_as_pycocotools()
|
|
41
|
+
|
|
42
|
+
from pycocotools.coco import COCO
|
|
43
|
+
from pycocotools.cocoeval import COCOeval
|
|
44
|
+
|
|
45
|
+
anno = COCO(str(anno_json)) # init annotations api
|
|
46
|
+
pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path)
|
|
47
|
+
|
|
48
|
+
val = COCOeval(anno, pred, "bbox")
|
|
49
|
+
val.evaluate()
|
|
50
|
+
val.accumulate()
|
|
51
|
+
val.summarize()
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
|
|
18
55
|
## Faster-COCO-Eval base
|
|
19
56
|
|
|
20
57
|
This package wraps a facebook C++ implementation of COCO-eval operations found in the
|
|
@@ -27,9 +64,9 @@ for coco's AP metrics, especially when dealing with a high number of instances i
|
|
|
27
64
|
For our use case with a test dataset of 5000 images from the coco val dataset.
|
|
28
65
|
Testing was carried out using the mmdetection framework and the eval_metric.py script. The indicators are presented below.
|
|
29
66
|
|
|
30
|
-
Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison
|
|
31
|
-
[
|
|
32
|
-
|
|
67
|
+
Visualization of testing **colab_example.ipynb** available in directory [examples/comparison](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison)
|
|
68
|
+
- [mmdet example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/mmdet/colab_example.ipynb)
|
|
69
|
+
- [ultralytics example](https://nbviewer.org/github/MiXaiLL76/faster_coco_eval/blob/main/examples/comparison/ultralytics/colab_example.ipynb)
|
|
33
70
|
|
|
34
71
|
### Summary for 5000 imgs
|
|
35
72
|
|
{faster_coco_eval-1.5.6 → faster_coco_eval-1.6.0}/csrc/faster_eval_api/coco_eval/cocoeval.cpp
RENAMED
|
@@ -11,6 +11,20 @@ namespace coco_eval
|
|
|
11
11
|
|
|
12
12
|
namespace COCOeval
|
|
13
13
|
{
|
|
14
|
+
template <typename T>
|
|
15
|
+
int v_index(const std::vector<T> &v, const T &key)
|
|
16
|
+
{
|
|
17
|
+
auto itr = std::find(v.begin(), v.end(), key);
|
|
18
|
+
|
|
19
|
+
if (itr != v.cend())
|
|
20
|
+
{
|
|
21
|
+
return std::distance(v.begin(), itr);
|
|
22
|
+
}
|
|
23
|
+
else
|
|
24
|
+
{
|
|
25
|
+
return -1;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
14
28
|
|
|
15
29
|
// Sort detections from highest score to lowest, such that
|
|
16
30
|
// detection_instances[detection_sorted_indices[t]] >=
|
|
@@ -44,7 +58,7 @@ namespace coco_eval
|
|
|
44
58
|
ignores->reserve(ground_truth_instances.size());
|
|
45
59
|
for (auto o : ground_truth_instances)
|
|
46
60
|
{
|
|
47
|
-
ignores->
|
|
61
|
+
ignores->emplace_back(
|
|
48
62
|
o.ignore || o.area < area_range[0] || o.area > area_range[1]);
|
|
49
63
|
}
|
|
50
64
|
|
|
@@ -84,9 +98,6 @@ namespace coco_eval
|
|
|
84
98
|
std::vector<int64_t> &ground_truth_matches = results->ground_truth_matches;
|
|
85
99
|
ground_truth_matches.resize(num_iou_thresholds * num_ground_truth, 0);
|
|
86
100
|
|
|
87
|
-
std::vector<int64_t> &ground_truth_orig_id = results->ground_truth_orig_id;
|
|
88
|
-
ground_truth_orig_id.resize(num_iou_thresholds * num_ground_truth, -1);
|
|
89
|
-
|
|
90
101
|
std::vector<int64_t> &detection_matches = results->detection_matches;
|
|
91
102
|
|
|
92
103
|
std::vector<bool> &detection_ignores = results->detection_ignores;
|
|
@@ -141,7 +152,12 @@ namespace coco_eval
|
|
|
141
152
|
ground_truth_instances[ground_truth_sorted_indices[match]].id;
|
|
142
153
|
ground_truth_matches[t * num_ground_truth + match] =
|
|
143
154
|
detection_instances[detection_sorted_indices[d]].id;
|
|
144
|
-
|
|
155
|
+
|
|
156
|
+
results->matched_annotations.push_back(
|
|
157
|
+
MatchedAnnotation(
|
|
158
|
+
ground_truth_matches[t * num_ground_truth + match], // DT_ID
|
|
159
|
+
detection_matches[t * num_detections + d], // GT_ID
|
|
160
|
+
best_iou));
|
|
145
161
|
}
|
|
146
162
|
|
|
147
163
|
// set unmatched detections outside of area range to ignore
|
|
@@ -280,9 +296,9 @@ namespace coco_eval
|
|
|
280
296
|
d < (int)evaluation.detection_scores.size() && d < max_detections;
|
|
281
297
|
++d)
|
|
282
298
|
{ // detected instances
|
|
283
|
-
evaluation_indices->
|
|
284
|
-
image_detection_indices->
|
|
285
|
-
detection_scores->
|
|
299
|
+
evaluation_indices->emplace_back(evaluation_index + i);
|
|
300
|
+
image_detection_indices->emplace_back(d);
|
|
301
|
+
detection_scores->emplace_back(evaluation.detection_scores[d]);
|
|
286
302
|
}
|
|
287
303
|
for (auto ground_truth_ignore : evaluation.ground_truth_ignores)
|
|
288
304
|
{
|
|
@@ -373,13 +389,13 @@ namespace coco_eval
|
|
|
373
389
|
|
|
374
390
|
const double recall =
|
|
375
391
|
static_cast<double>(true_positives_sum) / num_valid_ground_truth;
|
|
376
|
-
recalls->
|
|
392
|
+
recalls->emplace_back(recall);
|
|
377
393
|
const int64_t num_valid_detections =
|
|
378
394
|
true_positives_sum + false_positives_sum;
|
|
379
395
|
const double precision = num_valid_detections > 0
|
|
380
396
|
? static_cast<double>(true_positives_sum) / num_valid_detections
|
|
381
397
|
: 0.0;
|
|
382
|
-
precisions->
|
|
398
|
+
precisions->emplace_back(precision);
|
|
383
399
|
}
|
|
384
400
|
|
|
385
401
|
(*recalls_out)[recalls_out_index] = !recalls->empty() ? recalls->back() : 0;
|
|
@@ -544,14 +560,26 @@ namespace coco_eval
|
|
|
544
560
|
|
|
545
561
|
int evaluations_size = static_cast<int>(evaluations.size());
|
|
546
562
|
|
|
547
|
-
std::
|
|
548
|
-
|
|
549
|
-
std::vector<int64_t> out_ground_truth_orig_id = {};
|
|
563
|
+
std::unordered_map<std::string, double> matched;
|
|
564
|
+
|
|
550
565
|
for (auto eval : evaluations)
|
|
551
566
|
{
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
567
|
+
for (auto matched_annotation : eval.matched_annotations)
|
|
568
|
+
{
|
|
569
|
+
std::string key = std::to_string(matched_annotation.dt_id) + "_" + std::to_string(matched_annotation.gt_id);
|
|
570
|
+
|
|
571
|
+
if (matched.find(key) != matched.end())
|
|
572
|
+
{
|
|
573
|
+
if (matched[key] < matched_annotation.iou)
|
|
574
|
+
{
|
|
575
|
+
matched[key] = matched_annotation.iou;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
else
|
|
579
|
+
{
|
|
580
|
+
matched[key] = matched_annotation.iou;
|
|
581
|
+
}
|
|
582
|
+
}
|
|
555
583
|
}
|
|
556
584
|
|
|
557
585
|
std::vector<int64_t> counts = {
|
|
@@ -573,16 +601,14 @@ namespace coco_eval
|
|
|
573
601
|
"counts"_a = counts,
|
|
574
602
|
"date"_a = py::str(buffer),
|
|
575
603
|
|
|
604
|
+
"matched"_a = matched,
|
|
605
|
+
|
|
576
606
|
// precision and scores are num_iou_thresholds X num_recall_thresholds X num_categories X num_area_ranges X num_max_detections
|
|
577
607
|
"precision"_a = py::array(precisions_out.size(), precisions_out.data()).reshape(counts),
|
|
578
608
|
"scores"_a = py::array(scores_out.size(), scores_out.data()).reshape(counts),
|
|
579
609
|
|
|
580
610
|
// recall is num_iou_thresholds X num_categories X num_area_ranges X num_max_detections
|
|
581
611
|
"recall"_a = py::array(recalls_out.size(), recalls_out.data()).reshape(recall_counts),
|
|
582
|
-
|
|
583
|
-
"detection_matches"_a = py::array(out_detection_matches.size(), out_detection_matches.data()).reshape(matches_shape),
|
|
584
|
-
"ground_truth_matches"_a = py::array(out_ground_truth_matches.size(), out_ground_truth_matches.data()).reshape(matches_shape),
|
|
585
|
-
"ground_truth_orig_id"_a = py::array(out_ground_truth_orig_id.size(), out_ground_truth_orig_id.data()).reshape(matches_shape),
|
|
586
612
|
"evaluations_size"_a = evaluations_size);
|
|
587
613
|
}
|
|
588
614
|
|
|
@@ -601,6 +627,244 @@ namespace coco_eval
|
|
|
601
627
|
std::vector<ImageEvaluation> result = EvaluateImages(area_ranges, max_detections.back(), iou_thresholds, image_category_ious, image_category_ground_truth_instances, image_category_detection_instances);
|
|
602
628
|
return Accumulate(params, result);
|
|
603
629
|
}
|
|
630
|
+
|
|
631
|
+
void Dataset::append(int64_t img_id, int64_t cat_id, py::dict ann)
|
|
632
|
+
{
|
|
633
|
+
this->data[{img_id, cat_id}].emplace_back(ann);
|
|
634
|
+
}
|
|
635
|
+
std::vector<py::dict> Dataset::get(const int64_t &img_id, const int64_t &cat_id)
|
|
636
|
+
{
|
|
637
|
+
std::pair<int64_t, int64_t> key(img_id, cat_id);
|
|
638
|
+
|
|
639
|
+
if (this->data.find(key) != this->data.end())
|
|
640
|
+
{
|
|
641
|
+
return this->data[key];
|
|
642
|
+
}
|
|
643
|
+
else
|
|
644
|
+
{
|
|
645
|
+
return std::vector<py::dict>();
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
InstanceAnnotation parseInstanceAnnotation(const py::dict &ann)
|
|
650
|
+
{
|
|
651
|
+
uint64_t id = 0;
|
|
652
|
+
double score = 0.;
|
|
653
|
+
double area = 0.;
|
|
654
|
+
bool is_crowd = false;
|
|
655
|
+
bool ignore = false;
|
|
656
|
+
bool lvis_mark = false;
|
|
657
|
+
|
|
658
|
+
for (auto it : ann)
|
|
659
|
+
{
|
|
660
|
+
std::string key = it.first.cast<std::string>();
|
|
661
|
+
|
|
662
|
+
if (key == "id")
|
|
663
|
+
{
|
|
664
|
+
id = it.second.cast<uint64_t>();
|
|
665
|
+
}
|
|
666
|
+
else if (key == "score")
|
|
667
|
+
{
|
|
668
|
+
score = it.second.cast<double>();
|
|
669
|
+
}
|
|
670
|
+
else if (key == "area")
|
|
671
|
+
{
|
|
672
|
+
area = it.second.cast<double>();
|
|
673
|
+
}
|
|
674
|
+
else if ((key == "is_crowd") || key == "iscrowd")
|
|
675
|
+
{
|
|
676
|
+
is_crowd = it.second.cast<bool>();
|
|
677
|
+
}
|
|
678
|
+
else if (key == "ignore")
|
|
679
|
+
{
|
|
680
|
+
ignore = it.second.cast<bool>();
|
|
681
|
+
}
|
|
682
|
+
else if (key == "lvis_mark")
|
|
683
|
+
{
|
|
684
|
+
lvis_mark = it.second.cast<bool>();
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
return InstanceAnnotation(id, score, area, is_crowd, ignore, lvis_mark);
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
std::vector<InstanceAnnotation> Dataset::get_cpp_annotations(
|
|
691
|
+
const int64_t &img_id, const int64_t &cat_id)
|
|
692
|
+
{
|
|
693
|
+
std::vector<py::dict> anns = get(img_id, cat_id);
|
|
694
|
+
std::vector<InstanceAnnotation> result;
|
|
695
|
+
for (size_t i = 0; i < anns.size(); i++)
|
|
696
|
+
{
|
|
697
|
+
result.push_back(parseInstanceAnnotation(anns[i]));
|
|
698
|
+
}
|
|
699
|
+
return result;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
std::vector<std::vector<std::vector<InstanceAnnotation>>> Dataset::get_cpp_instances(
|
|
703
|
+
const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats)
|
|
704
|
+
{
|
|
705
|
+
std::vector<std::vector<std::vector<InstanceAnnotation>>> result;
|
|
706
|
+
for (size_t i = 0; i < img_ids.size(); i++)
|
|
707
|
+
{
|
|
708
|
+
int64_t img_id = img_ids[i];
|
|
709
|
+
result.push_back(std::vector<std::vector<InstanceAnnotation>>());
|
|
710
|
+
if (!useCats)
|
|
711
|
+
{
|
|
712
|
+
result[i].push_back(std::vector<InstanceAnnotation>());
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
for (size_t j = 0; j < cat_ids.size(); j++)
|
|
716
|
+
{
|
|
717
|
+
int64_t cat_id = cat_ids[j];
|
|
718
|
+
|
|
719
|
+
std::vector<InstanceAnnotation> anns = this->get_cpp_annotations(img_id, cat_id);
|
|
720
|
+
|
|
721
|
+
if (useCats)
|
|
722
|
+
{
|
|
723
|
+
result[i].emplace_back(anns);
|
|
724
|
+
}
|
|
725
|
+
else
|
|
726
|
+
{
|
|
727
|
+
result[i][0].insert(result[i][0].end(), anns.begin(), anns.end());
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
return result;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
std::vector<std::vector<std::vector<py::dict>>> Dataset::get_instances(
|
|
735
|
+
const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats)
|
|
736
|
+
{
|
|
737
|
+
std::vector<std::vector<std::vector<py::dict>>> result;
|
|
738
|
+
for (size_t i = 0; i < img_ids.size(); i++)
|
|
739
|
+
{
|
|
740
|
+
int64_t img_id = img_ids[i];
|
|
741
|
+
result.push_back(std::vector<std::vector<py::dict>>());
|
|
742
|
+
if (!useCats)
|
|
743
|
+
{
|
|
744
|
+
result[i].push_back(std::vector<py::dict>());
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
for (size_t j = 0; j < cat_ids.size(); j++)
|
|
748
|
+
{
|
|
749
|
+
int64_t cat_id = cat_ids[j];
|
|
750
|
+
|
|
751
|
+
std::vector<py::dict> anns = this->get(img_id, cat_id);
|
|
752
|
+
|
|
753
|
+
if (useCats)
|
|
754
|
+
{
|
|
755
|
+
result[i].emplace_back(anns);
|
|
756
|
+
}
|
|
757
|
+
else
|
|
758
|
+
{
|
|
759
|
+
result[i][0].insert(result[i][0].end(), anns.begin(), anns.end());
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
return result;
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
long double calc_auc(const std::vector<long double> &recall_list, const std::vector<long double> &precision_list)
|
|
767
|
+
{
|
|
768
|
+
std::vector<long double> mpre = precision_list;
|
|
769
|
+
|
|
770
|
+
for (size_t i = mpre.size() - 1; i > 0; i--)
|
|
771
|
+
{
|
|
772
|
+
mpre[i - 1] = std::max(mpre[i - 1], mpre[i]);
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
long double result = 0;
|
|
776
|
+
|
|
777
|
+
for (size_t i = 1; i < recall_list.size(); i++)
|
|
778
|
+
{
|
|
779
|
+
if (recall_list[i - 1] != recall_list[i])
|
|
780
|
+
{
|
|
781
|
+
result += (recall_list[i] - recall_list[i - 1]) * mpre[i];
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
return result;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
long double _summarize(const int &ap, const double &iouThr, const std::string &areaRng, const int &maxDet, const std::vector<int> &catIds, const py::object ¶ms, const std::vector<size_t> &counts, const py::object &nums_array)
|
|
789
|
+
{
|
|
790
|
+
std::vector<std::string> areaRngLbl = list_to_vec<std::string>(params.attr("areaRngLbl"));
|
|
791
|
+
std::vector<int> maxDets = list_to_vec<int>(params.attr("maxDets"));
|
|
792
|
+
std::vector<double> iouThrs = list_to_vec<double>(params.attr("iouThrs"));
|
|
793
|
+
|
|
794
|
+
std::vector<int> _catIds;
|
|
795
|
+
|
|
796
|
+
int iou_ind = v_index(iouThrs, iouThr);
|
|
797
|
+
int aind = v_index(areaRngLbl, areaRng);
|
|
798
|
+
int mind = v_index(maxDets, maxDet);
|
|
799
|
+
|
|
800
|
+
if (catIds.size() == 0)
|
|
801
|
+
{
|
|
802
|
+
for (size_t category = 0; category < counts[2]; category++)
|
|
803
|
+
{
|
|
804
|
+
_catIds.emplace_back(category);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
else
|
|
808
|
+
{
|
|
809
|
+
_catIds = catIds;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
std::vector<long double> result;
|
|
813
|
+
|
|
814
|
+
if (ap == 1)
|
|
815
|
+
{
|
|
816
|
+
// # dimension of precision: [TxRxKxAxM]
|
|
817
|
+
std::vector<std::vector<std::vector<std::vector<std::vector<double>>>>> precision = nums_array.cast<std::vector<std::vector<std::vector<std::vector<std::vector<double>>>>>>();
|
|
818
|
+
if (iou_ind != -1)
|
|
819
|
+
{
|
|
820
|
+
precision = {precision[iou_ind]};
|
|
821
|
+
}
|
|
822
|
+
for (size_t iou_threshold = 0; iou_threshold < precision.size(); iou_threshold++)
|
|
823
|
+
{
|
|
824
|
+
for (size_t recall_threshold = 0; recall_threshold < counts[1]; recall_threshold++)
|
|
825
|
+
{
|
|
826
|
+
for (const auto &category : _catIds)
|
|
827
|
+
{
|
|
828
|
+
long double pre = (long double)precision[iou_threshold][recall_threshold][category][aind][mind];
|
|
829
|
+
if (pre != -1)
|
|
830
|
+
{
|
|
831
|
+
result.emplace_back(pre);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
else
|
|
838
|
+
{
|
|
839
|
+
// # dimension of recall: [TxKxAxM]
|
|
840
|
+
std::vector<std::vector<std::vector<std::vector<double>>>> recall = nums_array.cast<std::vector<std::vector<std::vector<std::vector<double>>>>>();
|
|
841
|
+
if (iou_ind != -1)
|
|
842
|
+
{
|
|
843
|
+
recall = {recall[iou_ind]};
|
|
844
|
+
}
|
|
845
|
+
for (size_t iou_threshold = 0; iou_threshold < recall.size(); iou_threshold++)
|
|
846
|
+
{
|
|
847
|
+
for (const auto &category : catIds)
|
|
848
|
+
{
|
|
849
|
+
long double rec = (long double)recall[iou_threshold][category][aind][mind];
|
|
850
|
+
if (rec != -1)
|
|
851
|
+
{
|
|
852
|
+
result.emplace_back(rec);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
if (result.size() > 0)
|
|
859
|
+
{
|
|
860
|
+
return std::accumulate(result.begin(), result.end(), 0.0) / result.size();
|
|
861
|
+
}
|
|
862
|
+
else
|
|
863
|
+
{
|
|
864
|
+
return -1;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
604
868
|
} // namespace COCOeval
|
|
605
869
|
|
|
606
870
|
} // namespace coco_eval
|
|
@@ -34,6 +34,18 @@ namespace coco_eval
|
|
|
34
34
|
bool lvis_mark = false;
|
|
35
35
|
};
|
|
36
36
|
|
|
37
|
+
// Stores the match between a detected instance and a ground truth instance
|
|
38
|
+
struct MatchedAnnotation
|
|
39
|
+
{
|
|
40
|
+
MatchedAnnotation(
|
|
41
|
+
uint64_t dt_id,
|
|
42
|
+
uint64_t gt_id,
|
|
43
|
+
double iou) : dt_id{dt_id}, gt_id{gt_id}, iou{iou} {}
|
|
44
|
+
uint64_t dt_id;
|
|
45
|
+
uint64_t gt_id;
|
|
46
|
+
double iou;
|
|
47
|
+
};
|
|
48
|
+
|
|
37
49
|
// Stores intermediate results for evaluating detection results for a single
|
|
38
50
|
// image that has D detected instances and G ground truth instances. This stores
|
|
39
51
|
// matches between detected and ground truth instances
|
|
@@ -44,7 +56,6 @@ namespace coco_eval
|
|
|
44
56
|
std::vector<int64_t> detection_matches;
|
|
45
57
|
|
|
46
58
|
std::vector<int64_t> ground_truth_matches;
|
|
47
|
-
std::vector<int64_t> ground_truth_orig_id;
|
|
48
59
|
// The detection score of each of the D detected instances
|
|
49
60
|
std::vector<double> detection_scores;
|
|
50
61
|
|
|
@@ -55,6 +66,49 @@ namespace coco_eval
|
|
|
55
66
|
// Marks whether or not each of D instances was ignored from evaluation (e.g.,
|
|
56
67
|
// because it's outside aRng)
|
|
57
68
|
std::vector<bool> detection_ignores;
|
|
69
|
+
|
|
70
|
+
std::vector<MatchedAnnotation> matched_annotations;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
template <class T>
|
|
74
|
+
inline void hash_combine(std::size_t &seed, const T &v)
|
|
75
|
+
{
|
|
76
|
+
std::hash<T> hasher;
|
|
77
|
+
seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
struct hash_pair
|
|
81
|
+
{
|
|
82
|
+
std::size_t operator()(const std::pair<int64_t, int64_t> &p) const
|
|
83
|
+
{
|
|
84
|
+
std::size_t h = 0;
|
|
85
|
+
hash_combine(h, p.first);
|
|
86
|
+
hash_combine(h, p.second);
|
|
87
|
+
return h;
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
class Dataset
|
|
92
|
+
{
|
|
93
|
+
public:
|
|
94
|
+
Dataset()
|
|
95
|
+
{
|
|
96
|
+
this->data.reserve(8192); // RESERVING SPACE BEFOREHAND
|
|
97
|
+
// this->data.max_load_factor(0.25); // DECREASING MAX_LOAD_FACTOR
|
|
98
|
+
}
|
|
99
|
+
void append(int64_t img_id, int64_t cat_id, py::dict ann);
|
|
100
|
+
std::vector<py::dict> get(const int64_t &img_id, const int64_t &cat_id);
|
|
101
|
+
std::vector<InstanceAnnotation> get_cpp_annotations(
|
|
102
|
+
const int64_t &img_id, const int64_t &cat_id);
|
|
103
|
+
|
|
104
|
+
std::vector<std::vector<std::vector<InstanceAnnotation>>> get_cpp_instances(
|
|
105
|
+
const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats);
|
|
106
|
+
|
|
107
|
+
std::vector<std::vector<std::vector<py::dict>>> get_instances(
|
|
108
|
+
const std::vector<int64_t> &img_ids, const std::vector<int64_t> &cat_ids, const bool &useCats);
|
|
109
|
+
|
|
110
|
+
private:
|
|
111
|
+
std::unordered_map<std::pair<int64_t, int64_t>, std::vector<py::dict>, hash_pair> data;
|
|
58
112
|
};
|
|
59
113
|
|
|
60
114
|
template <class T>
|
|
@@ -100,5 +154,7 @@ namespace coco_eval
|
|
|
100
154
|
const ImageCategoryInstances<InstanceAnnotation> &
|
|
101
155
|
image_category_detection_instances);
|
|
102
156
|
|
|
157
|
+
long double calc_auc(const std::vector<long double> &recall_list, const std::vector<long double> &precision_list);
|
|
158
|
+
long double _summarize(const int &ap, const double &iouThr, const std::string &areaRng, const int &maxDet, const std::vector<int> &catIds, const py::object ¶ms, const std::vector<size_t> &counts, const py::object &nums_array);
|
|
103
159
|
} // namespace COCOeval
|
|
104
160
|
} // namespace coco_eval
|
|
@@ -49,8 +49,22 @@ namespace coco_eval
|
|
|
49
49
|
m.def("COCOevalAccumulate", &COCOeval::Accumulate, "COCOeval::Accumulate");
|
|
50
50
|
m.def("COCOevalEvaluateImages", &COCOeval::EvaluateImages, "COCOeval::EvaluateImages");
|
|
51
51
|
m.def("COCOevalEvaluateAccumulate", &COCOeval::EvaluateAccumulate, "COCOeval::EvaluateAccumulate");
|
|
52
|
-
|
|
52
|
+
|
|
53
|
+
// slow!
|
|
54
|
+
m.def("_summarize", &COCOeval::_summarize, "COCOeval::_summarize");
|
|
55
|
+
m.def("calc_auc", &COCOeval::calc_auc, "COCOeval::calc_auc");
|
|
56
|
+
|
|
57
|
+
pybind11::class_<COCOeval::InstanceAnnotation>(m, "InstanceAnnotation")
|
|
58
|
+
.def(pybind11::init<uint64_t, double, double, bool, bool, bool>());
|
|
59
|
+
|
|
53
60
|
pybind11::class_<COCOeval::ImageEvaluation>(m, "ImageEvaluation").def(pybind11::init<>());
|
|
61
|
+
|
|
62
|
+
pybind11::class_<COCOeval::Dataset>(m, "Dataset").def(pybind11::init<>())
|
|
63
|
+
.def("append", &COCOeval::Dataset::append)
|
|
64
|
+
.def("get", &COCOeval::Dataset::get)
|
|
65
|
+
.def("get_instances", &COCOeval::Dataset::get_instances)
|
|
66
|
+
.def("get_cpp_annotations", &COCOeval::Dataset::get_cpp_annotations)
|
|
67
|
+
.def("get_cpp_instances", &COCOeval::Dataset::get_cpp_instances);
|
|
54
68
|
|
|
55
69
|
#ifdef VERSION_INFO
|
|
56
70
|
m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO);
|