faster-coco-eval 1.6.4__tar.gz → 1.6.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. {faster_coco_eval-1.6.4/faster_coco_eval.egg-info → faster_coco_eval-1.6.5}/PKG-INFO +1 -1
  2. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/faster_eval_api/coco_eval/cocoeval.cpp +29 -1
  3. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/faster_eval_api/coco_eval/cocoeval.h +7 -1
  4. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/faster_eval_api/faster_eval_api.cpp +41 -3
  5. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/core/coco.py +24 -23
  6. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/core/cocoeval.py +92 -38
  7. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/core/faster_eval_api.py +47 -31
  8. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/version.py +1 -1
  9. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5/faster_coco_eval.egg-info}/PKG-INFO +1 -1
  10. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval.egg-info/SOURCES.txt +3 -1
  11. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/pyproject.toml +1 -0
  12. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_basic.py +7 -7
  13. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_boundary.py +3 -3
  14. faster_coco_eval-1.6.5/tests/test_crowdpose.py +53 -0
  15. faster_coco_eval-1.6.5/tests/test_world_evalutor.py +139 -0
  16. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/LICENSE +0 -0
  17. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/MANIFEST.in +0 -0
  18. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/README.md +0 -0
  19. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/mask_api/mask_api.cpp +0 -0
  20. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/mask_api/src/mask.cpp +0 -0
  21. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/mask_api/src/mask.h +0 -0
  22. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/csrc/mask_api/src/rle.cpp +0 -0
  23. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/__init__.py +0 -0
  24. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/core/__init__.py +0 -0
  25. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/core/mask.py +0 -0
  26. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/extra/__init__.py +0 -0
  27. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/extra/curves.py +0 -0
  28. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/extra/display.py +0 -0
  29. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/extra/draw.py +0 -0
  30. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/extra/extra.py +0 -0
  31. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval/extra/utils.py +0 -0
  32. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval.egg-info/dependency_links.txt +0 -0
  33. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval.egg-info/requires.txt +0 -0
  34. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/faster_coco_eval.egg-info/top_level.txt +0 -0
  35. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/requirements/extra.txt +0 -0
  36. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/requirements/runtime.txt +0 -0
  37. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/requirements/tests.txt +0 -0
  38. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/setup.cfg +0 -0
  39. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/setup.py +0 -0
  40. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_coco_metric.py +0 -0
  41. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_init_pycocotools.py +0 -0
  42. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_keypoints.py +0 -0
  43. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_lvis_metric.py +0 -0
  44. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_mask_api.py +0 -0
  45. {faster_coco_eval-1.6.4 → faster_coco_eval-1.6.5}/tests/test_torchmetrics.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: faster-coco-eval
3
- Version: 1.6.4
3
+ Version: 1.6.5
4
4
  Summary: Faster interpretation of the original COCOEval
5
5
  Home-page: https://github.com/MiXaiLL76/faster_coco_eval
6
6
  Author: MiXaiLL76
@@ -629,6 +629,34 @@ namespace coco_eval
629
629
  {
630
630
  this->data[{img_id, cat_id}].emplace_back(ann);
631
631
  }
632
+
633
+ void Dataset::clean(){
634
+ this->data.clear();
635
+ }
636
+
637
+ py::tuple Dataset::make_tuple() const {
638
+ std::vector<std::pair<int64_t, int64_t>> keys;
639
+ std::vector<std::vector<py::dict>> values;
640
+
641
+ for (const auto& [key, value] : this->data) {
642
+ keys.push_back(key);
643
+ values.push_back(value);
644
+ }
645
+ return py::make_tuple(keys, values);
646
+ }
647
+
648
+ void Dataset::load_tuple(py::tuple data){
649
+ if (data.size() != 2)
650
+ throw std::runtime_error("Invalid state!");
651
+
652
+ std::vector<std::pair<int64_t, int64_t>> keys = data[0].cast<std::vector<std::pair<int64_t, int64_t>>>();
653
+ std::vector<std::vector<py::dict>> values = data[1].cast<std::vector<std::vector<py::dict>>>();
654
+
655
+ for (size_t i = 0; i < keys.size(); i++) {
656
+ this->data[keys[i]] = values[i];
657
+ }
658
+ }
659
+
632
660
  std::vector<py::dict> Dataset::get(const int64_t &img_id, const int64_t &cat_id)
633
661
  {
634
662
  std::pair<int64_t, int64_t> key(img_id, cat_id);
@@ -864,4 +892,4 @@ namespace coco_eval
864
892
 
865
893
  } // namespace COCOeval
866
894
 
867
- } // namespace coco_eval
895
+ } // namespace coco_eval
@@ -97,6 +97,12 @@ namespace coco_eval
97
97
  // this->data.max_load_factor(0.25); // DECREASING MAX_LOAD_FACTOR
98
98
  }
99
99
  void append(int64_t img_id, int64_t cat_id, py::dict ann);
100
+ void clean();
101
+
102
+ // pickle
103
+ py::tuple make_tuple() const ;
104
+ void load_tuple(py::tuple data);
105
+
100
106
  std::vector<py::dict> get(const int64_t &img_id, const int64_t &cat_id);
101
107
  std::vector<InstanceAnnotation> get_cpp_annotations(
102
108
  const int64_t &img_id, const int64_t &cat_id);
@@ -157,4 +163,4 @@ namespace coco_eval
157
163
  long double calc_auc(const std::vector<long double> &recall_list, const std::vector<long double> &precision_list);
158
164
  long double _summarize(const int &ap, const double &iouThr, const std::string &areaRng, const int &maxDet, const std::vector<int> &catIds, const py::object &params, const std::vector<size_t> &counts, const py::object &nums_array);
159
165
  } // namespace COCOeval
160
- } // namespace coco_eval
166
+ } // namespace coco_eval
@@ -57,14 +57,52 @@ namespace coco_eval
57
57
  pybind11::class_<COCOeval::InstanceAnnotation>(m, "InstanceAnnotation")
58
58
  .def(pybind11::init<uint64_t, double, double, bool, bool, bool>());
59
59
 
60
- pybind11::class_<COCOeval::ImageEvaluation>(m, "ImageEvaluation").def(pybind11::init<>());
60
+ pybind11::class_<COCOeval::ImageEvaluation>(m, "ImageEvaluation").def(pybind11::init<>())
61
+ .def(py::pickle(
62
+ [](const COCOeval::ImageEvaluation &p) {
63
+
64
+ std::vector<std::tuple<uint64_t, uint64_t, double>> matched_annotations;
65
+ for (size_t i = 0; i < p.matched_annotations.size(); i++) {
66
+ matched_annotations.push_back(std::make_tuple(p.matched_annotations[i].dt_id, p.matched_annotations[i].gt_id, p.matched_annotations[i].iou));
67
+ }
68
+
69
+ return py::make_tuple(p.detection_matches, p.ground_truth_matches, p.detection_scores, p.ground_truth_ignores, p.detection_ignores, matched_annotations);
70
+ },
71
+ [](py::tuple t) { // __setstate__
72
+ if (t.size() != 6)
73
+ throw std::runtime_error("Invalid state!");
74
+
75
+ COCOeval::ImageEvaluation p;
76
+ p.detection_matches = t[0].cast<std::vector<int64_t>>();
77
+ p.ground_truth_matches = t[1].cast<std::vector<int64_t>>();
78
+ p.detection_scores = t[2].cast<std::vector<double>>();
79
+ p.ground_truth_ignores = t[3].cast<std::vector<bool>>();
80
+ p.detection_ignores = t[4].cast<std::vector<bool>>();
81
+ std::vector<std::tuple<uint64_t, uint64_t, double>> matched_annotations = t[5].cast<std::vector<std::tuple<uint64_t, uint64_t, double>>>();
82
+ for (size_t i = 0; i < matched_annotations.size(); i++) {
83
+ p.matched_annotations.emplace_back(std::get<0>(matched_annotations[i]), std::get<1>(matched_annotations[i]), std::get<2>(matched_annotations[i]));
84
+ }
85
+ return p;
86
+ }
87
+ ));
61
88
 
62
89
  pybind11::class_<COCOeval::Dataset>(m, "Dataset").def(pybind11::init<>())
63
90
  .def("append", &COCOeval::Dataset::append)
91
+ .def("clean", &COCOeval::Dataset::clean)
64
92
  .def("get", &COCOeval::Dataset::get)
65
93
  .def("get_instances", &COCOeval::Dataset::get_instances)
66
94
  .def("get_cpp_annotations", &COCOeval::Dataset::get_cpp_annotations)
67
- .def("get_cpp_instances", &COCOeval::Dataset::get_cpp_instances);
95
+ .def("get_cpp_instances", &COCOeval::Dataset::get_cpp_instances)
96
+ .def(py::pickle(
97
+ [](const COCOeval::Dataset &p) {
98
+ return p.make_tuple();
99
+ },
100
+ [](py::tuple t) { // __setstate__
101
+ COCOeval::Dataset p;
102
+ p.load_tuple(t);
103
+ return p;
104
+ }
105
+ ));
68
106
 
69
107
  #ifdef VERSION_INFO
70
108
  m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO);
@@ -73,4 +111,4 @@ namespace coco_eval
73
111
  #endif
74
112
  }
75
113
 
76
- } // namespace coco_eval
114
+ } // namespace coco_eval
@@ -9,7 +9,7 @@ import pathlib
9
9
  import time
10
10
  import warnings
11
11
  from collections import defaultdict
12
- from typing import List, Optional, Union
12
+ from typing import Callable, List, Optional, Union
13
13
 
14
14
  import numpy as np
15
15
 
@@ -28,6 +28,7 @@ class COCO:
28
28
  self,
29
29
  annotation_file: Optional[Union[str, dict, os.PathLike, pathlib.PosixPath]] = None,
30
30
  use_deepcopy: bool = False,
31
+ print_function: Callable = logger.debug,
31
32
  ):
32
33
  """Constructor of Microsoft COCO helper class.
33
34
 
@@ -45,21 +46,17 @@ class COCO:
45
46
  {},
46
47
  )
47
48
  self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list)
48
- self.print_function = logger.debug
49
+ self.print_function = print_function
49
50
  self.use_deepcopy = use_deepcopy
51
+ self.annotation_file = annotation_file
50
52
 
51
53
  if annotation_file is not None:
52
54
  self._print_function("loading annotations into memory...")
53
55
  tic = time.time()
54
- if type(annotation_file) in [str, os.PathLike, pathlib.PosixPath]:
55
- self.dataset = self.load_json(annotation_file)
56
- elif type(annotation_file) is dict:
57
- if self.use_deepcopy:
58
- self.dataset = copy.deepcopy(annotation_file)
59
- else:
60
- self.dataset = annotation_file.copy()
56
+ if type(annotation_file) in [str, os.PathLike, pathlib.PosixPath, dict, list]:
57
+ self.dataset = self.load_json(annotation_file, self.use_deepcopy)
61
58
  else:
62
- self.dataset = None
59
+ raise TypeError(f"type {type(annotation_file)} is not supported")
63
60
 
64
61
  assert type(self.dataset) is dict, f"annotation file format {type(self.dataset)} not supported"
65
62
  self.print_function(f"Done (t={time.time() - tic:0.2f}s)")
@@ -290,29 +287,36 @@ class COCO:
290
287
  return [self.imgs[ids]]
291
288
 
292
289
  @staticmethod
293
- def load_json(json_file: Optional[Union[str, os.PathLike]]) -> dict:
290
+ def load_json(
291
+ json_file: Union[str, os.PathLike, pathlib.PosixPath, dict, list], use_deepcopy: Optional[bool] = False
292
+ ) -> dict:
294
293
  """Load a json file.
295
294
 
296
295
  Args:
297
- json_file (str or os.PathLike): Path to the json file
296
+ json_file (str or os.PathLike or dict or list): Path to the json file or data dict
298
297
 
299
298
  Return:
300
299
  data (dict): Loaded json data
301
300
  """
302
-
303
- with open(json_file) as io:
304
- _data = json.load(io)
301
+ if type(json_file) in [str, os.PathLike]:
302
+ with open(json_file) as io:
303
+ _data = json.load(io)
304
+ else:
305
+ if use_deepcopy:
306
+ return copy.deepcopy(json_file)
307
+ else:
308
+ return json_file.copy()
305
309
  return _data
306
310
 
307
311
  def loadRes(
308
312
  self,
309
- resFile: Union[str, os.PathLike, np.ndarray],
313
+ resFile: Union[str, os.PathLike, pathlib.PosixPath, dict, list, np.ndarray],
310
314
  min_score: float = 0.0,
311
315
  ) -> "COCO":
312
316
  """Load result file and return a result api object.
313
317
 
314
318
  Args:
315
- resFile (str) : file name of result file
319
+ resFile (str or os.PathLike or dict or list) : file name of result file
316
320
  min_score (float) : minimum score to consider a result
317
321
 
318
322
  Return:
@@ -324,15 +328,12 @@ class COCO:
324
328
 
325
329
  self.print_function("Loading and preparing results...")
326
330
  tic = time.time()
327
- if type(resFile) in [str, os.PathLike]:
328
- anns = self.load_json(resFile)
331
+ if type(resFile) in [str, os.PathLike, pathlib.PosixPath, dict, list]:
332
+ anns = self.load_json(resFile, self.use_deepcopy)
329
333
  elif type(resFile) is np.ndarray:
330
334
  anns = self.loadNumpyAnnotations(resFile)
331
335
  else:
332
- if self.use_deepcopy:
333
- anns = copy.deepcopy(resFile)
334
- else:
335
- anns = resFile.copy()
336
+ raise TypeError(f"type {type(resFile)} is not supported")
336
337
 
337
338
  assert type(anns) is list, "results in not an array of objects"
338
339
 
@@ -10,7 +10,7 @@ from typing import Callable, List, Optional, Union
10
10
  if sys.version_info >= (3, 8):
11
11
  from typing import Literal
12
12
 
13
- iouTypeT = Literal["segm", "bbox", "keypoints", "boundary"]
13
+ iouTypeT = Literal["segm", "bbox", "keypoints", "keypoints_crowd", "boundary"]
14
14
  else:
15
15
  iouTypeT = str
16
16
 
@@ -30,9 +30,10 @@ class COCOeval:
30
30
  cocoGt: Optional[COCO] = None,
31
31
  cocoDt: Optional[COCO] = None,
32
32
  iouType: iouTypeT = "segm",
33
- print_function: Callable = logger.debug,
33
+ print_function: Callable = logger.info,
34
34
  extra_calc: bool = False,
35
35
  kpt_oks_sigmas: Optional[List[float]] = None,
36
+ use_area: Optional[bool] = True,
36
37
  lvis_style: bool = False,
37
38
  separate_eval: bool = False,
38
39
  boundary_dilation_ratio: float = 0.02,
@@ -53,6 +54,8 @@ class COCOeval:
53
54
  whether to perform extra calculations, defaults to False
54
55
  kpt_oks_sigmas (None or list):
55
56
  list of sigmas for keypoint evaluation, defaults to None
57
+ use_area (bool):
58
+ If gt annotations (eg. CrowdPose) do not have 'area', please set use_area=False.
56
59
  lvis_style (bool):
57
60
  whether to use LVIS style evaluation, defaults to False
58
61
  separate_eval (bool):
@@ -81,6 +84,7 @@ class COCOeval:
81
84
  self.separate_eval = separate_eval
82
85
  self.boundary_dilation_ratio = boundary_dilation_ratio
83
86
  self.boundary_cpu_count = boundary_cpu_count
87
+ self.use_area = use_area
84
88
 
85
89
  if iouType == "keypoints" and self.lvis_style:
86
90
  logger.warning("lvis_style not supported for keypoint evaluation")
@@ -119,6 +123,10 @@ class COCOeval:
119
123
  params."""
120
124
  p = self.params
121
125
 
126
+ # cleanup before create new
127
+ self.gt_dataset.clean()
128
+ self.dt_dataset.clean()
129
+
122
130
  cat_ids = p.catIds if p.catIds else None
123
131
 
124
132
  gts = self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=cat_ids))
@@ -128,8 +136,8 @@ class COCOeval:
128
136
  for gt in gts:
129
137
  gt["ignore"] = gt["ignore"] if "ignore" in gt else 0
130
138
  gt["ignore"] = "iscrowd" in gt and gt["iscrowd"]
131
- if p.iouType == "keypoints":
132
- gt["ignore"] = (gt.get("num_keypoints") == 0) or gt["ignore"]
139
+ if "keypoints" in p.iouType:
140
+ gt["ignore"] = (gt.get("num_keypoints", 0) == 0) or gt["ignore"]
133
141
 
134
142
  img_pl = defaultdict(set) # per image list of categories present in image
135
143
  img_nl = {} # per image map of categories not present in image
@@ -159,7 +167,6 @@ class COCOeval:
159
167
  if img_sizes.get(image_id) is None:
160
168
  t = dataset.imgs[image_id]
161
169
  img_sizes[image_id] = t["height"], t["width"]
162
- return img_sizes[image_id]
163
170
 
164
171
  for gt in gts:
165
172
  if p.compute_rle:
@@ -328,7 +335,13 @@ class COCOeval:
328
335
  z = np.zeros(k)
329
336
  dx = np.max((z, x0 - xd), axis=0) + np.max((z, xd - x1), axis=0)
330
337
  dy = np.max((z, y0 - yd), axis=0) + np.max((z, yd - y1), axis=0)
331
- e = (dx**2 + dy**2) / vars / (gt["area"] + np.spacing(1)) / 2
338
+
339
+ if self.use_area:
340
+ e = (dx**2 + dy**2) / vars / (gt["area"] + np.spacing(1)) / 2
341
+ else:
342
+ tmparea = gt["bbox"][3] * gt["bbox"][2] * 0.53
343
+ e = (dx**2 + dy**2) / vars / (tmparea + np.spacing(1)) / 2
344
+
332
345
  if k1 > 0:
333
346
  e = e[vg > 0]
334
347
  ious[i, j] = np.sum(np.exp(-e)) / e.shape[0]
@@ -362,30 +375,12 @@ class COCOeval:
362
375
  iouStr = f"{p.iouThrs[0]:0.2f}:{p.iouThrs[-1]:0.2f}" if iouThr is None else f"{iouThr:0.2f}"
363
376
 
364
377
  if catIds is not None:
365
- # catNames = [
366
- # self.cocoGt.cats[p.catIds[used_cat_idx]].get(
367
- # "name", used_cat_idx
368
- # )
369
- # for used_cat_idx in catIds
370
- # ]
371
378
  freq_str = f"catIds=={str(catIds):>3s}"
372
379
 
373
380
  if self.lvis_style and (freq_group_idx is not None):
374
381
  catIds = self.freq_groups[freq_group_idx]
375
382
  freq_str = f"catIds={p.imgCountLbl[freq_group_idx]:>3s}"
376
383
 
377
- # In my case, C++ too slow...
378
- # mean_s = _C._summarize(
379
- # ap,
380
- # (iouThr if iouThr else -1),
381
- # areaRng,
382
- # maxDets,
383
- # catIds,
384
- # p,
385
- # self.eval["counts"],
386
- # self.eval["precision"] if ap == 1 else self.eval["recall"],
387
- # )
388
-
389
384
  aind = p.areaRngLbl.index(areaRng)
390
385
  mind = p.maxDets.index(maxDets)
391
386
 
@@ -440,6 +435,7 @@ class COCOeval:
440
435
  def _summarizeDets():
441
436
  _count = 17 if self.lvis_style else 14
442
437
  stats = np.zeros((_count,))
438
+
443
439
  stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) # AP_all
444
440
  stats[1] = self._summarize(1, iouThr=0.5, maxDets=self.params.maxDets[-1]) # AP_50
445
441
  stats[2] = self._summarize(1, iouThr=0.75, maxDets=self.params.maxDets[-1]) # AP_75
@@ -452,8 +448,7 @@ class COCOeval:
452
448
  stats[15] = self._summarize(1, maxDets=self.params.maxDets[-1], freq_group_idx=1) # APc
453
449
  stats[16] = self._summarize(1, maxDets=self.params.maxDets[-1], freq_group_idx=2) # APf
454
450
 
455
- # AR_first or AR_all
456
- stats[6] = self._summarize(0, maxDets=self.params.maxDets[0])
451
+ stats[6] = self._summarize(0, maxDets=self.params.maxDets[0]) # AR_first or AR_all
457
452
  if len(self.params.maxDets) >= 2:
458
453
  stats[7] = self._summarize(0, maxDets=self.params.maxDets[1]) # AR_second
459
454
  if len(self.params.maxDets) >= 3:
@@ -470,16 +465,42 @@ class COCOeval:
470
465
 
471
466
  def _summarizeKps():
472
467
  stats = np.zeros((10,))
473
- stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1])
474
- stats[1] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.5)
475
- stats[2] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.75)
476
- stats[3] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="medium")
477
- stats[4] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="large")
478
- stats[5] = self._summarize(0, maxDets=self.params.maxDets[-1])
479
- stats[6] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.5)
480
- stats[7] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.75)
481
- stats[8] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="medium")
482
- stats[9] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="large")
468
+ stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) # AP_all
469
+ stats[1] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.5) # AP_50
470
+ stats[2] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.75) # AP_75
471
+ stats[3] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="medium") # AP_medium
472
+ stats[4] = self._summarize(1, maxDets=self.params.maxDets[-1], areaRng="large") # AP_large
473
+ stats[5] = self._summarize(0, maxDets=self.params.maxDets[-1]) # AR_all
474
+ stats[6] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.5) # AR_50
475
+ stats[7] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.75) # AR_75
476
+ stats[8] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="medium") # AR_medium
477
+ stats[9] = self._summarize(0, maxDets=self.params.maxDets[-1], areaRng="large") # AR_large
478
+ return stats
479
+
480
+ def _summarizeKps_crowd():
481
+ stats = np.zeros((9,))
482
+ stats[0] = self._summarize(1, maxDets=self.params.maxDets[-1]) # AP_all
483
+ stats[1] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.5) # AP_50
484
+ stats[2] = self._summarize(1, maxDets=self.params.maxDets[-1], iouThr=0.75) # AP_75
485
+ stats[3] = self._summarize(0, maxDets=self.params.maxDets[-1]) # AR_all
486
+ stats[4] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.5) # AR_50
487
+ stats[5] = self._summarize(0, maxDets=self.params.maxDets[-1], iouThr=0.75) # AR_75
488
+ type_result = self.get_type_result(first=0.2, second=0.8)
489
+
490
+ p = self.params
491
+ iStr = " {:<18} {} @[ IoU={:<9} | type={:>6s} | maxDets={:>3d} ] = {:0.3f}"
492
+ titleStr = "Average Precision"
493
+ typeStr = "(AP)"
494
+ iouStr = f"{p.iouThrs[0]:0.2f}:{p.iouThrs[-1]:0.2f}"
495
+ self.print_function(iStr.format(titleStr, typeStr, iouStr, "easy", self.params.maxDets[-1], type_result[0]))
496
+ self.print_function(
497
+ iStr.format(titleStr, typeStr, iouStr, "medium", self.params.maxDets[-1], type_result[1])
498
+ )
499
+ self.print_function(iStr.format(titleStr, typeStr, iouStr, "hard", self.params.maxDets[-1], type_result[2]))
500
+ stats[6] = type_result[0] # AP_easy
501
+ stats[7] = type_result[1] # AP_medium
502
+ stats[8] = type_result[2] # AP_hard
503
+
483
504
  return stats
484
505
 
485
506
  if not self.eval:
@@ -491,12 +512,45 @@ class COCOeval:
491
512
  summarize = _summarizeDets
492
513
  elif iouType == "keypoints":
493
514
  summarize = _summarizeKps
515
+ elif iouType == "keypoints_crowd":
516
+ summarize = _summarizeKps_crowd
494
517
  else:
495
- ValueError(f"iouType must be bbox, segm, boundary or keypoints. Get {iouType}")
518
+ ValueError(f"iouType must be bbox, segm, boundary or keypoints or keypoints_crowd. Get {iouType}")
496
519
 
497
520
  self.all_stats = summarize()
498
521
  self.stats = self.all_stats[:12]
499
522
 
523
+ def get_type_result(self, first=0.01, second=0.85):
524
+ easy, mid, hard = self.split(self.cocoGt.annotation_file, first, second)
525
+ res = []
526
+ prev_print = self.print_function
527
+ self.print_function = lambda *args, **kwargs: None
528
+
529
+ for curr_type in [easy, mid, hard]:
530
+ curr_list = curr_type
531
+ self.params.imgIds = curr_list
532
+ self.evaluate()
533
+ self.accumulate()
534
+ score = self.eval["precision"][:, :, :, 0, :]
535
+ res.append(round(np.mean(score), 4))
536
+
537
+ self.print_function = prev_print
538
+ return res
539
+
540
+ def split(self, gt_file, first=0.01, second=0.85):
541
+ data = COCO.load_json(gt_file, use_deepcopy=True)
542
+ easy = []
543
+ mid = []
544
+ hard = []
545
+ for item in data["images"]:
546
+ if item["crowdIndex"] < first:
547
+ easy.append(item["id"])
548
+ elif item["crowdIndex"] < second:
549
+ mid.append(item["id"])
550
+ else:
551
+ hard.append(item["id"])
552
+ return easy, mid, hard
553
+
500
554
  def __str__(self):
501
555
  self.summarize()
502
556
  return str(self.__repr__())
@@ -574,7 +628,7 @@ class Params:
574
628
 
575
629
  if iouType in set(["segm", "bbox", "boundary"]):
576
630
  self.setDetParams()
577
- elif iouType == "keypoints":
631
+ elif "keypoints" in iouType:
578
632
  self.setKpParams()
579
633
  if kpt_sigmas is not None:
580
634
  self.kpt_oks_sigmas = np.array(kpt_sigmas)
@@ -48,7 +48,7 @@ class COCOeval_faster(COCOevalBase):
48
48
 
49
49
  if p.iouType in set(["segm", "bbox", "boundary"]):
50
50
  computeIoU = self.computeIoU
51
- elif p.iouType == "keypoints":
51
+ elif "keypoints" in p.iouType:
52
52
  computeIoU = self.computeOks
53
53
  else:
54
54
  raise ValueError(f"p.iouType must be segm, bbox, boundary or keypoints. Get {p.iouType}")
@@ -178,28 +178,54 @@ class COCOeval_faster(COCOevalBase):
178
178
 
179
179
  @property
180
180
  def stats_as_dict(self):
181
- labels = [
182
- "AP_all",
183
- "AP_50",
184
- "AP_75",
185
- "AP_small",
186
- "AP_medium",
187
- "AP_large",
188
- "AR_all",
189
- "AR_second",
190
- "AR_third",
191
- "AR_small",
192
- "AR_medium",
193
- "AR_large",
194
- ]
195
-
196
181
  if self.params.iouType in set(["segm", "bbox", "boundary"]):
197
- labels += ["AR_50", "AR_75"]
198
- else:
199
- labels = [label for label in labels if "small" not in label]
182
+ labels = [
183
+ "AP_all",
184
+ "AP_50",
185
+ "AP_75",
186
+ "AP_small",
187
+ "AP_medium",
188
+ "AP_large",
189
+ "AR_all",
190
+ "AR_second",
191
+ "AR_third",
192
+ "AR_small",
193
+ "AR_medium",
194
+ "AR_large",
195
+ "AR_50",
196
+ "AR_75",
197
+ ]
200
198
 
201
- if self.lvis_style:
202
- labels += ["APr", "APc", "APf"]
199
+ if self.lvis_style:
200
+ labels += ["APr", "APc", "APf"]
201
+
202
+ elif self.params.iouType == "keypoints":
203
+ labels = [
204
+ "AP_all",
205
+ "AP_50",
206
+ "AP_75",
207
+ "AP_medium",
208
+ "AP_large",
209
+ "AR_all",
210
+ "AR_50",
211
+ "AR_75",
212
+ "AR_medium",
213
+ "AR_large",
214
+ ]
215
+ elif self.params.iouType == "keypoints_crowd":
216
+ labels = [
217
+ "AP_all",
218
+ "AP_50",
219
+ "AP_75",
220
+ "AR_all",
221
+ "AR_50",
222
+ "AR_75",
223
+ "AP_easy",
224
+ "AP_medium",
225
+ "AP_hard",
226
+ ]
227
+ else:
228
+ ValueError(f"iouType must be bbox, segm, boundary or keypoints. Get {self.params.iouType}")
203
229
 
204
230
  if self.matched:
205
231
  labels += [
@@ -207,16 +233,6 @@ class COCOeval_faster(COCOevalBase):
207
233
  "mAUC_" + str(int(self.params.iouThrs[0] * 100)),
208
234
  ]
209
235
 
210
- maxDets = self.params.maxDets
211
- if len(maxDets) > 1:
212
- labels[6] = f"AR_{maxDets[0]}"
213
-
214
- if len(maxDets) >= 2:
215
- labels[7] = f"AR_{maxDets[1]}"
216
-
217
- if len(maxDets) >= 3:
218
- labels[8] = f"AR_{maxDets[2]}"
219
-
220
236
  return {_label: float(self.all_stats[i]) for i, _label in enumerate(labels)}
221
237
 
222
238
  @staticmethod
@@ -1,2 +1,2 @@
1
- __version__ = "1.6.4"
1
+ __version__ = "1.6.5"
2
2
  __author__ = "MiXaiLL76"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: faster-coco-eval
3
- Version: 1.6.4
3
+ Version: 1.6.5
4
4
  Summary: Faster interpretation of the original COCOEval
5
5
  Home-page: https://github.com/MiXaiLL76/faster_coco_eval
6
6
  Author: MiXaiLL76
@@ -34,8 +34,10 @@ requirements/tests.txt
34
34
  tests/test_basic.py
35
35
  tests/test_boundary.py
36
36
  tests/test_coco_metric.py
37
+ tests/test_crowdpose.py
37
38
  tests/test_init_pycocotools.py
38
39
  tests/test_keypoints.py
39
40
  tests/test_lvis_metric.py
40
41
  tests/test_mask_api.py
41
- tests/test_torchmetrics.py
42
+ tests/test_torchmetrics.py
43
+ tests/test_world_evalutor.py
@@ -33,3 +33,4 @@ lint.select = [
33
33
  lint.mccabe.max-complexity = 10
34
34
  # Use Google-style docstrings.
35
35
  lint.pydocstyle.convention = "google"
36
+ exclude = ["examples"]
@@ -68,7 +68,7 @@ class TestBaseCoco(unittest.TestCase):
68
68
  self.assertEqual(kp_params.iou_type, "segm")
69
69
 
70
70
  def test_bad_coco_set(self):
71
- with self.assertRaises(AssertionError):
71
+ with self.assertRaises(TypeError):
72
72
  COCO(1)
73
73
 
74
74
  def test_bad_iou_type(self):
@@ -86,9 +86,9 @@ class TestBaseCoco(unittest.TestCase):
86
86
  "AP_small": -1.0,
87
87
  "AP_medium": 0.7099009900990099,
88
88
  "AP_large": -1.0,
89
- "AR_1": 0.0,
90
- "AR_10": 0.45384615384615384,
91
- "AR_100": 0.7153846153846154,
89
+ "AR_all": 0.0,
90
+ "AR_second": 0.45384615384615384,
91
+ "AR_third": 0.7153846153846154,
92
92
  "AR_small": -1.0,
93
93
  "AR_medium": 0.7153846153846154,
94
94
  "AR_large": -1.0,
@@ -119,9 +119,9 @@ class TestBaseCoco(unittest.TestCase):
119
119
  "AP_small": -1.0,
120
120
  "AP_medium": 0.7367986798679867,
121
121
  "AP_large": 0.0,
122
- "AR_1": 0.6666666666666666,
123
- "AR_10": 0.75,
124
- "AR_100": 0.75,
122
+ "AR_all": 0.6666666666666666,
123
+ "AR_second": 0.75,
124
+ "AR_third": 0.75,
125
125
  "AR_small": -1.0,
126
126
  "AR_medium": 0.7916666666666666,
127
127
  "AR_large": 0.0,
@@ -103,9 +103,9 @@ class TestBoundary(unittest.TestCase):
103
103
  "AP_small": -1.0,
104
104
  "AP_medium": 0.7367986798679867,
105
105
  "AP_large": 0.0,
106
- "AR_1": 0.6666666666666666,
107
- "AR_10": 0.75,
108
- "AR_100": 0.75,
106
+ "AR_all": 0.6666666666666666,
107
+ "AR_second": 0.75,
108
+ "AR_third": 0.75,
109
109
  "AR_small": -1.0,
110
110
  "AR_medium": 0.7916666666666666,
111
111
  "AR_large": 0.0,
@@ -0,0 +1,53 @@
1
+ #!/usr/bin/python3
2
+ import os
3
+ import unittest
4
+
5
+ import numpy as np
6
+
7
+ from faster_coco_eval import COCO, COCOeval_faster
8
+
9
+
10
+ class TestCrowdpose(unittest.TestCase):
11
+ """Test Crowdpose functionality."""
12
+
13
+ def setUp(self):
14
+ self.gt_file = os.path.join("dataset", "example_crowdpose_val.json")
15
+ self.dt_file = os.path.join("dataset", "example_crowdpose_preds.json")
16
+ self.sigmas = (
17
+ np.array([0.79, 0.79, 0.72, 0.72, 0.62, 0.62, 1.07, 1.07, 0.87, 0.87, 0.89, 0.89, 0.79, 0.79]) / 10.0
18
+ )
19
+
20
+ if not os.path.exists(self.gt_file):
21
+ self.gt_file = os.path.join(os.path.dirname(__file__), self.gt_file)
22
+ self.dt_file = os.path.join(os.path.dirname(__file__), self.dt_file)
23
+
24
+ def test_crowdpose_eval(self):
25
+ stats_as_dict = {
26
+ "AP_all": 0.7877215935879303,
27
+ "AP_50": 0.9881188118811886,
28
+ "AP_75": 0.7314356435643564,
29
+ "AR_all": 0.8222222222222223,
30
+ "AR_50": 1.0,
31
+ "AR_75": 0.7777777777777778,
32
+ "AP_easy": 1.0,
33
+ "AP_medium": 0.9802,
34
+ "AP_hard": 0.4116,
35
+ }
36
+
37
+ cocoGt = COCO(self.gt_file)
38
+
39
+ cocoDt = cocoGt.loadRes(self.dt_file)
40
+
41
+ cocoEval = COCOeval_faster(
42
+ cocoGt, cocoDt, iouType="keypoints_crowd", kpt_oks_sigmas=self.sigmas, use_area=False
43
+ )
44
+
45
+ cocoEval.evaluate()
46
+ cocoEval.accumulate()
47
+ cocoEval.summarize()
48
+
49
+ self.assertAlmostEqual(cocoEval.stats_as_dict, stats_as_dict, places=10)
50
+
51
+
52
+ if __name__ == "__main__":
53
+ unittest.main()
@@ -0,0 +1,139 @@
1
+ #!/usr/bin/python3
2
+ import concurrent.futures
3
+ import copy
4
+ import itertools
5
+ import os
6
+ import unittest
7
+ from collections import defaultdict
8
+
9
+ import numpy as np
10
+
11
+ import faster_coco_eval.core.mask as mask_util
12
+ from faster_coco_eval import COCO, COCOeval_faster
13
+
14
+
15
+ def _evaluate(coco_eval_proc: COCOeval_faster, anns: list):
16
+ coco_eval_proc.params.imgIds = [ann["image_id"] for ann in anns]
17
+ coco_eval_proc.cocoDt = coco_eval_proc.cocoGt.loadRes(anns)
18
+ coco_eval_proc.evaluate()
19
+ return coco_eval_proc._evalImgs_cpp, coco_eval_proc.params.imgIds
20
+
21
+
22
+ class TestWorldCoco(unittest.TestCase):
23
+ """Test basic rankX COCO functionality."""
24
+
25
+ def setUp(self):
26
+ self.gt_file = os.path.join("dataset", "gt_dataset.json")
27
+ self.dt_file = os.path.join("dataset", "dt_dataset.json")
28
+ self.gt_lvis_file = os.path.join("lvis_dataset", "lvis_val_100.json")
29
+ self.dt_lvis_file = os.path.join("lvis_dataset", "lvis_results_100.json")
30
+
31
+ if not os.path.exists(self.gt_file):
32
+ self.gt_file = os.path.join(os.path.dirname(__file__), self.gt_file)
33
+ self.dt_file = os.path.join(os.path.dirname(__file__), self.dt_file)
34
+ self.gt_lvis_file = os.path.join(os.path.dirname(__file__), self.gt_lvis_file)
35
+ self.dt_lvis_file = os.path.join(os.path.dirname(__file__), self.dt_lvis_file)
36
+
37
+ prepared_anns = COCO.load_json(self.dt_file)
38
+
39
+ cocoGt = COCO(self.gt_file)
40
+ self.prepared_anns_droped_bbox = [
41
+ {
42
+ "image_id": ann["image_id"],
43
+ "category_id": ann["category_id"],
44
+ "iscrowd": ann["iscrowd"],
45
+ "id": ann["id"],
46
+ "score": ann["score"],
47
+ "segmentation": mask_util.merge(
48
+ mask_util.frPyObjects(
49
+ ann["segmentation"],
50
+ cocoGt.imgs[ann["image_id"]]["height"],
51
+ cocoGt.imgs[ann["image_id"]]["width"],
52
+ )
53
+ ),
54
+ }
55
+ for ann in prepared_anns
56
+ ]
57
+
58
+ self.prepared_anns_droped_bbox_by_image_id = defaultdict(list)
59
+ for ann in self.prepared_anns_droped_bbox:
60
+ self.prepared_anns_droped_bbox_by_image_id[ann["image_id"]].append(copy.deepcopy(ann))
61
+
62
+ def test_world(self):
63
+ # MULTI
64
+ coco_eval_rank = COCOeval_faster(COCO(self.gt_file), iouType="segm", separate_eval=True)
65
+
66
+ eval_imgs = []
67
+ eval_img_ids = []
68
+
69
+ with concurrent.futures.ProcessPoolExecutor(len(self.prepared_anns_droped_bbox_by_image_id)) as executor:
70
+ for evalImgs, imgIds in executor.map(
71
+ _evaluate, itertools.repeat(coco_eval_rank), self.prepared_anns_droped_bbox_by_image_id.values()
72
+ ):
73
+ eval_imgs.append(evalImgs)
74
+ eval_img_ids += imgIds
75
+
76
+ coco_eval_rank.params.imgIds = eval_img_ids
77
+ coco_eval_rank._paramsEval = copy.deepcopy(coco_eval_rank.params)
78
+
79
+ coco_eval_rank._evalImgs_cpp = np.array(eval_imgs).T.ravel().tolist()
80
+
81
+ coco_eval_rank.accumulate()
82
+ coco_eval_rank.summarize()
83
+
84
+ # SOLO
85
+ coco_eval = COCOeval_faster(COCO(self.gt_file), iouType="segm")
86
+ coco_eval.cocoDt = coco_eval.cocoGt.loadRes(self.prepared_anns_droped_bbox)
87
+ coco_eval.evaluate()
88
+ coco_eval.accumulate()
89
+ coco_eval.summarize()
90
+
91
+ self.assertAlmostEqual(coco_eval.stats_as_dict, coco_eval_rank.stats_as_dict, 12)
92
+
93
+ def test_world_lvis(self):
94
+ # MULTI
95
+ coco_eval_rank = COCOeval_faster(COCO(self.gt_lvis_file), iouType="bbox", lvis_style=True, separate_eval=True)
96
+
97
+ prepared_anns = defaultdict(list)
98
+ for ann in COCO.load_json(self.dt_lvis_file):
99
+ prepared_anns[ann["image_id"]].append(copy.deepcopy(ann))
100
+
101
+ all_images_ids = list(prepared_anns.keys())
102
+ rank2_prepared_anns = defaultdict(list)
103
+
104
+ for key in all_images_ids[::2]:
105
+ rank2_prepared_anns["rank1"] += prepared_anns[key]
106
+
107
+ for key in all_images_ids[1::2]:
108
+ rank2_prepared_anns["rank2"] += prepared_anns[key]
109
+
110
+ eval_imgs = []
111
+ eval_img_ids = []
112
+
113
+ with concurrent.futures.ProcessPoolExecutor(2) as executor:
114
+ for evalImgs, imgIds in executor.map(
115
+ _evaluate, itertools.repeat(coco_eval_rank), rank2_prepared_anns.values()
116
+ ):
117
+ eval_imgs.append(evalImgs)
118
+ eval_img_ids += imgIds
119
+
120
+ coco_eval_rank.params.imgIds = eval_img_ids
121
+ coco_eval_rank._paramsEval = copy.deepcopy(coco_eval_rank.params)
122
+ coco_eval_rank.freq_groups = coco_eval_rank._prepare_freq_group()
123
+ coco_eval_rank._evalImgs_cpp = np.array(eval_imgs).T.ravel().tolist()
124
+
125
+ coco_eval_rank.accumulate()
126
+ coco_eval_rank.summarize()
127
+
128
+ # SOLO
129
+ coco_eval = COCOeval_faster(COCO(self.gt_lvis_file), iouType="bbox", lvis_style=True)
130
+ coco_eval.cocoDt = coco_eval.cocoGt.loadRes(self.dt_lvis_file)
131
+ coco_eval.evaluate()
132
+ coco_eval.accumulate()
133
+ coco_eval.summarize()
134
+
135
+ self.assertAlmostEqual(coco_eval.stats_as_dict, coco_eval_rank.stats_as_dict, 12)
136
+
137
+
138
+ if __name__ == "__main__":
139
+ unittest.main()