halpecocotools 0.0.0__cp312-cp312-macosx_14_0_arm64.whl

halpecocotools/cocoeval.py

@@ -0,0 +1,818 @@
+__author__ = 'tsungyi, Haoyi Zhu'
+
+import numpy as np
+import datetime
+import time
+from collections import defaultdict
+from . import mask as maskUtils
+import copy
+
+class COCOeval:
+    # Interface for evaluating detection on the Microsoft COCO dataset.
+    #
+    # The usage for CocoEval is as follows:
+    #  cocoGt=..., cocoDt=...       # load dataset and results
+    #  E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object
+    #  E.params.recThrs = ...;      # set parameters as desired
+    #  E.evaluate();                # run per image evaluation
+    #  E.accumulate();              # accumulate per image results
+    #  E.summarize();               # display summary metrics of results
+    # For example usage see evalDemo.m and http://mscoco.org/.
+    #
+    # The evaluation parameters are as follows (defaults in brackets):
+    #  imgIds     - [all] N img ids to use for evaluation
+    #  catIds     - [all] K cat ids to use for evaluation
+    #  iouThrs    - [.5:.05:.95] T=10 IoU thresholds for evaluation
+    #  recThrs    - [0:.01:1] R=101 recall thresholds for evaluation
+    #  areaRng    - [...] A=4 object area ranges for evaluation
+    #  maxDets    - [1 10 100] M=3 thresholds on max detections per image
+    #  iouType    - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints'
+    #  iouType replaced the now DEPRECATED useSegm parameter.
+    #  useCats    - [1] if true use category labels for evaluation
+    # Note: if useCats=0 category labels are ignored as in proposal scoring.
+    # Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified.
+    #
+    # evaluate(): evaluates detections on every image and every category and
+    # concats the results into the "evalImgs" with fields:
+    #  dtIds      - [1xD] id for each of the D detections (dt)
+    #  gtIds      - [1xG] id for each of the G ground truths (gt)
+    #  dtMatches  - [TxD] matching gt id at each IoU or 0
+    #  gtMatches  - [TxG] matching dt id at each IoU or 0
+    #  dtScores   - [1xD] confidence of each dt
+    #  gtIgnore   - [1xG] ignore flag for each gt
+    #  dtIgnore   - [TxD] ignore flag for each dt at each IoU
+    #
+    # accumulate(): accumulates the per-image, per-category evaluation
+    # results in "evalImgs" into the dictionary "eval" with fields:
+    #  params     - parameters used for evaluation
+    #  date       - date evaluation was performed
+    #  counts     - [T,R,K,A,M] parameter dimensions (see above)
+    #  precision  - [TxRxKxAxM] precision for every evaluation setting
+    #  recall     - [TxKxAxM] max recall for every evaluation setting
+    # Note: precision and recall==-1 for settings with no gt objects.
+    #
+    # See also coco, mask, pycocoDemo, pycocoEvalDemo
+    #
+    # The original Microsoft COCO Toolbox is written
+    # by Piotr Dollar and Tsung-Yi Lin, 2015.
+    # Licensed under the Simplified BSD License [see coco/license.txt]
+    # 
+    # Updated and renamed to Halpe COCO Toolbox (cocohalpetools) \
+    # by Haoyi Zhu in 2021. The Halpe COCO Toolbox is
+    # developed to support wholebody human pose datasets, 
+    # including Halpe Full-body and COCO WholeBody.
+    
+    def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'):
+        '''
+        Initialize CocoEval using coco APIs for gt and dt
+        :param cocoGt: coco object with ground truth annotations
+        :param cocoDt: coco object with detection results
+        :return: None
+        '''
+        if not iouType:
+            print('iouType not specified. use default iouType segm')
+        self.cocoGt   = cocoGt              # ground truth COCO API
+        self.cocoDt   = cocoDt              # detections COCO API
+        self.params   = {}                  # evaluation parameters
+        self.evalImgs = defaultdict(list)   # per-image per-category evaluation results [KxAxI] elements
+        self.eval     = {}                  # accumulated evaluation results
+        self._gts = defaultdict(list)       # gt for evaluation
+        self._dts = defaultdict(list)       # dt for evaluation
+        self.params = Params(iouType=iouType) # parameters
+        self._paramsEval = {}               # parameters for evaluation
+        self.stats = []                     # result summarization
+        self.ious = {}                      # ious between all gts and dts
+        if not cocoGt is None:
+            self.params.imgIds = sorted(cocoGt.getImgIds())
+            self.params.catIds = sorted(cocoGt.getCatIds())
+
+
+    def _prepare(self):
+        '''
+        Prepare ._gts and ._dts for evaluation based on params
+        :return: None
+        '''
+        def _toMask(anns, coco):
+            # modify ann['segmentation'] by reference
+            for ann in anns:
+                rle = coco.annToRLE(ann)
+                ann['segmentation'] = rle
+        p = self.params
+        if p.useCats:
+            gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds))
+            dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds))
+        else:
+            gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds))
+            dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds))
+
+        # convert ground truth to mask if iouType == 'segm'
+        if p.iouType == 'segm':
+            _toMask(gts, self.cocoGt)
+            _toMask(dts, self.cocoDt)
+        # set ignore flag
+        for gt in gts:
+            gt['ignore'] = gt['ignore'] if 'ignore' in gt else 0
+            # gt['ignore'] = 'iscrowd' in gt and gt['iscrowd']
+            if p.iouType == 'keypoints':
+                if len(gt['keypoints']) % 133 == 0 or len(gt['keypoints']) % 136 == 0:
+                    _ignore = gt['ignore']
+                    gt['ignore'] = dict()
+                    body_kp = np.array(gt['keypoints'][:-116*3])
+                    foot_kp = np.array(gt['keypoints'][-116*3:-110*3])
+                    face_kp = np.array(gt['keypoints'][-110*3:-42*3])
+                    hand_kp = np.array(gt['keypoints'][-42*3:])
+                    fullbody_kp = np.array(gt['keypoints'])
+
+                    vg_body = body_kp[2::3]
+                    vg_foot = foot_kp[2::3]
+                    vg_face = face_kp[2::3]
+                    vg_hand = hand_kp[2::3]
+                    vg_fullbody = fullbody_kp[2::3]
+                    k1_body = np.count_nonzero(vg_body > 0)
+                    k1_foot = np.count_nonzero(vg_foot > 0)
+                    k1_face = np.count_nonzero(vg_face > 0)
+                    k1_hand = np.count_nonzero(vg_hand > 0)
+                    k1_fullbody = np.count_nonzero(vg_fullbody > 0)
+
+                    gt['ignore']['body'] = (k1_body == 0) or _ignore
+                    gt['ignore']['foot'] = (k1_foot == 0) or _ignore
+                    gt['ignore']['face'] = (k1_face == 0) or _ignore
+                    gt['ignore']['hand'] = (k1_hand == 0) or _ignore
+                    gt['ignore']['fullbody'] = (k1_fullbody == 0) or _ignore
+                else:
+                    gt['ignore'] = (gt['num_keypoints'] == 0) or gt['ignore']
+        self._gts = defaultdict(list)       # gt for evaluation
+        self._dts = defaultdict(list)       # dt for evaluation
+        for gt in gts:
+            self._gts[gt['image_id'], gt['category_id']].append(gt)
+        for dt in dts:
+            vd = np.array(dt['keypoints'])[2::3]
+            if len(vd[vd>0]) == 0:
+                continue
+            self._dts[dt['image_id'], dt['category_id']].append(dt)
+        self.evalImgs = defaultdict(list)   # per-image per-category evaluation results
+        self.eval     = {}                  # accumulated evaluation results
+
+    def evaluate(self):
+        '''
+        Run per image evaluation on given images and store results (a list of dict) in self.evalImgs
+        :return: None
+        '''
+        tic = time.time()
+        print('Running per image evaluation...')
+        p = self.params
+        # add backward compatibility if useSegm is specified in params
+        if not p.useSegm is None:
+            p.iouType = 'segm' if p.useSegm == 1 else 'bbox'
+            print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType))
+        print('Evaluate annotation type *{}*'.format(p.iouType))
+        p.imgIds = list(np.unique(p.imgIds))
+        if p.useCats:
+            p.catIds = list(np.unique(p.catIds))
+        p.maxDets = sorted(p.maxDets)
+        self.params=p
+
+        self._prepare()
+        # loop through images, area range, max detection number
+        catIds = p.catIds if p.useCats else [-1]
+
+        if p.iouType == 'segm' or p.iouType == 'bbox':
+            computeIoU = self.computeIoU
+        elif p.iouType == 'keypoints':
+            computeIoU = self.computeOks
+        self.ious = {(imgId, catId): computeIoU(imgId, catId) \
+                        for imgId in p.imgIds
+                        for catId in catIds}
+
+        evaluateImg = self.evaluateImg
+        maxDet = p.maxDets[-1]
+        self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet)
+                 for catId in catIds
+                 for areaRng in p.areaRng
+                 for imgId in p.imgIds
+             ]
+        self._paramsEval = copy.deepcopy(self.params)
+        toc = time.time()
+        print('DONE (t={:0.2f}s).'.format(toc-tic))
+
+    def computeIoU(self, imgId, catId):
+        p = self.params
+        if p.useCats:
+            gt = self._gts[imgId,catId]
+            dt = self._dts[imgId,catId]
+        else:
+            gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]]
+            dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]]
+        if len(gt) == 0 and len(dt) ==0:
+            return []
+        inds = np.argsort([-d['score'] for d in dt], kind='mergesort')
+        dt = [dt[i] for i in inds]
+        if len(dt) > p.maxDets[-1]:
+            dt=dt[0:p.maxDets[-1]]
+
+        if p.iouType == 'segm':
+            g = [g['segmentation'] for g in gt]
+            d = [d['segmentation'] for d in dt]
+        elif p.iouType == 'bbox':
+            g = [g['bbox'] for g in gt]
+            d = [d['bbox'] for d in dt]
+        else:
+            raise Exception('unknown iouType for iou computation')
+
+        # compute iou between each dt and gt region
+        iscrowd = [int(o['iscrowd']) for o in gt]
+        ious = maskUtils.iou(d,g,iscrowd)
+        return ious
+
+    def computeOks(self, imgId, catId):
+        parts = ['body', 'foot', 'face', 'hand', 'fullbody']
+        _ious = {}
+        for part in parts:
+            p = self.params
+            # dimention here should be Nxm
+            gts = self._gts[imgId, catId]
+            dts = self._dts[imgId, catId]
+            inds = np.argsort([-d['score'] for d in dts], kind='mergesort')
+            dts = [dts[i] for i in inds]
+            if len(dts) > p.maxDets[-1]:
+                dts = dts[0:p.maxDets[-1]]
+            # if len(gts) == 0 and len(dts) == 0:
+            if len(gts) == 0 or len(dts) == 0:
+                _ious[part] = []
+                continue
+            ious = np.zeros((len(dts), len(gts)))
+            if len(gts[0]['keypoints']) == 136 * 3:
+                sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89, .8,.8,.8,.89, .89, .89, .89, .89, .89,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15])/10.0
+                body_kpnum = 20
+                foot_kpnum = 6
+                face_kpnum = 68
+                hand_kpnum = 21 * 2
+
+            elif len(gts[0]['keypoints']) == 133 * 3:   # surport for COCO WholeBody
+                sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89, .89, .89, .89, .89, .89, .89,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15,
+                         .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15])/10.0
+                body_kpnum = 17
+                foot_kpnum = 6
+                face_kpnum = 68
+                hand_kpnum = 21 * 2
+
+            if part == 'body':
+                sigmas = sigmas[0 : body_kpnum]
+            elif part == 'foot':
+                sigmas = sigmas[body_kpnum : body_kpnum + foot_kpnum]
+            elif part == 'face':
+                sigmas = sigmas[body_kpnum + foot_kpnum : body_kpnum + foot_kpnum + face_kpnum]
+            elif part == 'hand':
+                sigmas = sigmas[body_kpnum + foot_kpnum + face_kpnum:]
+
+            vars = (sigmas * 2)**2
+            k = len(sigmas)
+            from tkinter import _flatten
+            # compute oks between each detection and ground truth object
+            for j, gt in enumerate(gts):
+                # create bounds for ignore regions(double the gt bbox)
+                if part == 'body':
+                    g = np.array(gt['keypoints'][0 : body_kpnum*3],dtype=np.float32)
+                elif part == 'foot':
+                    g = np.array(gt['keypoints'][body_kpnum*3 : (body_kpnum + foot_kpnum)*3],dtype=np.float32)
+                elif part == 'face':
+                    g = np.array(gt['keypoints'][(body_kpnum + foot_kpnum)*3 : (body_kpnum + foot_kpnum + face_kpnum)*3],dtype=np.float32)
+                elif part == 'hand':
+                    g = np.array(gt['keypoints'][(body_kpnum + foot_kpnum + face_kpnum)*3:],dtype=np.float32)
+                else:
+                    g = np.array(gt['keypoints'],dtype=np.float32)
+
+                xg = g[0::3]; yg = g[1::3]; vg = g[2::3]
+                k1 = np.count_nonzero(vg > 0)
+                bb = gt['bbox']
+                x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2
+                y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2
+                for i, dt in enumerate(dts):
+                    if part == 'body':
+                        d = np.array(dt['keypoints'][0 : body_kpnum*3],dtype=np.float32)
+                    elif part == 'foot':
+                        d = np.array(dt['keypoints'][body_kpnum*3 : (body_kpnum + foot_kpnum)*3],dtype=np.float32)
+                    elif part == 'face':
+                        d = np.array(dt['keypoints'][(body_kpnum + foot_kpnum)*3 : (body_kpnum + foot_kpnum + face_kpnum)*3],dtype=np.float32)
+                    elif part == 'hand':
+                        d = np.array(dt['keypoints'][(body_kpnum + foot_kpnum + face_kpnum)*3:],dtype=np.float32)
+                    else:
+                        d = np.array(dt['keypoints'], dtype=np.float32)
+                    
+                    xd = d[0::3]; yd = d[1::3]
+                    if k1>0:
+                        # measure the per-keypoint distance if keypoints visible
+                        dx = xd - xg
+                        dy = yd - yg
+                    else:
+                        # measure minimum distance to keypoints in (x0,y0) & (x1,y1)
+                        z = np.zeros((k))
+                        dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0)
+                        dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0)
+                    e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2
+                    if k1 > 0:
+                        e=e[vg > 0]
+                    ious[i, j] = np.sum(np.exp(-e)) / e.shape[0]
+            _ious[part] = ious
+        return _ious
+
+    def evaluateImg(self, imgId, catId, aRng, maxDet):
+        '''
+        perform evaluation for single category and image
+        :return: dict (single image results)
+        '''
+        if self.params.iouType == 'keypoints':
+            parts = ['body', 'foot', 'face', 'hand', 'fullbody']
+            res = {}
+            for part in parts:
+                p = self.params
+                if p.useCats:
+                    gt = self._gts[imgId,catId]
+                    dt = self._dts[imgId,catId]
+                else:
+                    gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]]
+                    dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]]
+                if len(gt) == 0 and len(dt) ==0:
+                    res[part] = None
+                    continue
+
+                for g in gt:
+                    if g['ignore'][part] or (g['area']<aRng[0] or g['area']>aRng[1]):
+                        g['_ignore'] = 1
+                    else:
+                        g['_ignore'] = 0
+
+                # sort dt highest score first, sort gt ignore last
+                gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort')
+                gt = [gt[i] for i in gtind]
+                dtind = np.argsort([-d['score'] for d in dt], kind='mergesort')
+                dt = [dt[i] for i in dtind[0:maxDet]]
+                iscrowd = [int(o['iscrowd']) for o in gt]
+                # load computed ious
+                ious = self.ious[imgId, catId][part][:, gtind] if len(self.ious[imgId, catId][part]) > 0 else self.ious[imgId, catId][part]
+
+                T = len(p.iouThrs)
+                G = len(gt)
+                D = len(dt)
+                gtm  = np.zeros((T,G))
+                dtm  = np.zeros((T,D))
+                gtIg = np.array([g['_ignore'] for g in gt])
+                dtIg = np.zeros((T,D))
+                if not len(ious)==0:
+                    for tind, t in enumerate(p.iouThrs):
+                        for dind, d in enumerate(dt):
+                            # information about best match so far (m=-1 -> unmatched)
+                            iou = min([t,1-1e-10])
+                            m   = -1
+                            for gind, g in enumerate(gt):
+                                # if this gt already matched, and not a crowd, continue
+                                if gtm[tind,gind]>0 and not iscrowd[gind]:
+                                    continue
+                                # if dt matched to reg gt, and on ignore gt, stop
+                                if m>-1 and gtIg[m]==0 and gtIg[gind]==1:
+                                    break
+                                # continue to next gt unless better match made
+                                if ious[dind,gind] < iou:
+                                    continue
+                                # if match successful and best so far, store appropriately
+                                iou=ious[dind,gind]
+                                m=gind
+                            # if match made store id of match for both dt and gt
+                            if m ==-1:
+                                continue
+                            dtIg[tind,dind] = gtIg[m]
+                            dtm[tind,dind]  = gt[m]['id']
+                            gtm[tind,m]     = d['id']
+                # set unmatched detections outside of area range to ignore
+                a = np.array([d['area']<aRng[0] or d['area']>aRng[1] for d in dt]).reshape((1, len(dt)))
+                dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0)))
+                # store results for given image and category
+                res[part]={
+                        'image_id':     imgId,
+                        'category_id':  catId,
+                        'aRng':         aRng,
+                        'maxDet':       maxDet,
+                        'dtIds':        [d['id'] for d in dt],
+                        'gtIds':        [g['id'] for g in gt],
+                        'dtMatches':    dtm,
+                        'gtMatches':    gtm,
+                        'dtScores':     [d['score'] for d in dt],
+                        'gtIgnore':     gtIg,
+                        'dtIgnore':     dtIg,
+                    }
+            return res
+        else:
+            p = self.params
+            if p.useCats:
+                gt = self._gts[imgId,catId]
+                dt = self._dts[imgId,catId]
+            else:
+                gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]]
+                dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]]
+            if len(gt) == 0 and len(dt) ==0:
+                return None
+
+            for g in gt:
+                if g['ignore'] or (g['area']<aRng[0] or g['area']>aRng[1]):
+                    g['_ignore'] = 1
+                else:
+                    g['_ignore'] = 0
+
+            # sort dt highest score first, sort gt ignore last
+            gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort')
+            gt = [gt[i] for i in gtind]
+            dtind = np.argsort([-d['score'] for d in dt], kind='mergesort')
+            dt = [dt[i] for i in dtind[0:maxDet]]
+            iscrowd = [int(o['iscrowd']) for o in gt]
+            # load computed ious
+            ious = self.ious[imgId, catId][:, gtind] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId]
+
+            T = len(p.iouThrs)
+            G = len(gt)
+            D = len(dt)
+            gtm  = np.zeros((T,G))
+            dtm  = np.zeros((T,D))
+            gtIg = np.array([g['_ignore'] for g in gt])
+            dtIg = np.zeros((T,D))
+            if not len(ious)==0:
+                for tind, t in enumerate(p.iouThrs):
+                    for dind, d in enumerate(dt):
+                        # information about best match so far (m=-1 -> unmatched)
+                        iou = min([t,1-1e-10])
+                        m   = -1
+                        for gind, g in enumerate(gt):
+                            # if this gt already matched, and not a crowd, continue
+                            if gtm[tind,gind]>0 and not iscrowd[gind]:
+                                continue
+                            # if dt matched to reg gt, and on ignore gt, stop
+                            if m>-1 and gtIg[m]==0 and gtIg[gind]==1:
+                                break
+                            # continue to next gt unless better match made
+                            if ious[dind,gind] < iou:
+                                continue
+                            # if match successful and best so far, store appropriately
+                            iou=ious[dind,gind]
+                            m=gind
+                        # if match made store id of match for both dt and gt
+                        if m ==-1:
+                            continue
+                        dtIg[tind,dind] = gtIg[m]
+                        dtm[tind,dind]  = gt[m]['id']
+                        gtm[tind,m]     = d['id']
+            # set unmatched detections outside of area range to ignore
+            a = np.array([d['area']<aRng[0] or d['area']>aRng[1] for d in dt]).reshape((1, len(dt)))
+            dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0)))
+            # store results for given image and category
+            return {
+                    'image_id':     imgId,
+                    'category_id':  catId,
+                    'aRng':         aRng,
+                    'maxDet':       maxDet,
+                    'dtIds':        [d['id'] for d in dt],
+                    'gtIds':        [g['id'] for g in gt],
+                    'dtMatches':    dtm,
+                    'gtMatches':    gtm,
+                    'dtScores':     [d['score'] for d in dt],
+                    'gtIgnore':     gtIg,
+                    'dtIgnore':     dtIg,
+                }
+            return res
+
+    def accumulate(self, p = None):
+        '''
+        Accumulate per image evaluation results and store the result in self.eval
+        :param p: input params for evaluation
+        :return: None
+        '''
+        print('Accumulating evaluation results...')
+        tic = time.time()
+        if not self.evalImgs:
+            print('Please run evaluate() first')
+        if self.params.iouType == 'keypoints':
+            parts = ['body', 'foot', 'face', 'hand', 'fullbody']
+            for part in parts:
+                # allows input customized parameters
+                if p is None:
+                    p = self.params
+                p.catIds = p.catIds if p.useCats == 1 else [-1]
+                T           = len(p.iouThrs)
+                R           = len(p.recThrs)
+                K           = len(p.catIds) if p.useCats else 1
+                A           = len(p.areaRng)
+                M           = len(p.maxDets)
+                precision   = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories
+                recall      = -np.ones((T,K,A,M))
+                scores      = -np.ones((T,R,K,A,M))
+
+                # create dictionary for future indexing
+                _pe = self._paramsEval
+                catIds = _pe.catIds if _pe.useCats else [-1]
+                setK = set(catIds)
+                setA = set(map(tuple, _pe.areaRng))
+                setM = set(_pe.maxDets)
+                setI = set(_pe.imgIds)
+                # get inds to evaluate
+                k_list = [n for n, k in enumerate(p.catIds)  if k in setK]
+                m_list = [m for n, m in enumerate(p.maxDets) if m in setM]
+                a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA]
+                i_list = [n for n, i in enumerate(p.imgIds)  if i in setI]
+                I0 = len(_pe.imgIds)
+                A0 = len(_pe.areaRng)
+                # retrieve E at each category, area range, and max number of detections
+
+                for k, k0 in enumerate(k_list):
+                    Nk = k0*A0*I0
+                    for a, a0 in enumerate(a_list):
+                        Na = a0*I0
+                        for m, maxDet in enumerate(m_list):
+                            E = [self.evalImgs[Nk + Na + i][part] for i in i_list]
+                            E = [e for e in E if not e is None]
+                            if len(E) == 0:
+                                continue
+                            dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E])
+
+                            # different sorting method generates slightly different results.
+                            # mergesort is used to be consistent as Matlab implementation.
+                            inds = np.argsort(-dtScores, kind='mergesort')
+                            dtScoresSorted = dtScores[inds]
+
+                            dtm  = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds]
+                            dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet]  for e in E], axis=1)[:,inds]
+                            gtIg = np.concatenate([e['gtIgnore'] for e in E])
+                            npig = np.count_nonzero(gtIg==0 )
+                            if npig == 0:
+                                continue
+                            tps = np.logical_and(               dtm,  np.logical_not(dtIg) )
+                            fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) )
+
+                            tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float)
+                            fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float)
+                            for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):
+                                tp = np.array(tp)
+                                fp = np.array(fp)
+                                nd = len(tp)
+                                rc = tp / npig
+                                pr = tp / (fp+tp+np.spacing(1))
+                                q  = np.zeros((R,))
+                                ss = np.zeros((R,))
+
+                                if nd:
+                                    recall[t,k,a,m] = rc[-1]
+                                else:
+                                    recall[t,k,a,m] = 0
+
+                                # numpy is slow without cython optimization for accessing elements
+                                # use python array gets significant speed improvement
+                                pr = pr.tolist(); q = q.tolist()
+
+                                for i in range(nd-1, 0, -1):
+                                    if pr[i] > pr[i-1]:
+                                        pr[i-1] = pr[i]
+
+                                inds = np.searchsorted(rc, p.recThrs, side='left')
+                                try:
+                                    for ri, pi in enumerate(inds):
+                                        q[ri] = pr[pi]
+                                        ss[ri] = dtScoresSorted[pi]
+                                except:
+                                    pass
+                                precision[t,:,k,a,m] = np.array(q)
+                                scores[t,:,k,a,m] = np.array(ss)
+                self.eval[part] = {
+                    'params': p,
+                    'counts': [T, R, K, A, M],
+                    'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+                    'precision': precision,
+                    'recall':   recall,
+                    'scores': scores,
+                }
+        else:
+            if p is None:
+                p = self.params
+            p.catIds = p.catIds if p.useCats == 1 else [-1]
+            T           = len(p.iouThrs)
+            R           = len(p.recThrs)
+            K           = len(p.catIds) if p.useCats else 1
+            A           = len(p.areaRng)
+            M           = len(p.maxDets)
+            precision   = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories
+            recall      = -np.ones((T,K,A,M))
+            scores      = -np.ones((T,R,K,A,M))
+
+            # create dictionary for future indexing
+            _pe = self._paramsEval
+            catIds = _pe.catIds if _pe.useCats else [-1]
+            setK = set(catIds)
+            setA = set(map(tuple, _pe.areaRng))
+            setM = set(_pe.maxDets)
+            setI = set(_pe.imgIds)
+            # get inds to evaluate
+            k_list = [n for n, k in enumerate(p.catIds)  if k in setK]
+            m_list = [m for n, m in enumerate(p.maxDets) if m in setM]
+            a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA]
+            i_list = [n for n, i in enumerate(p.imgIds)  if i in setI]
+            I0 = len(_pe.imgIds)
+            A0 = len(_pe.areaRng)
+            # retrieve E at each category, area range, and max number of detections
+
+            for k, k0 in enumerate(k_list):
+                Nk = k0*A0*I0
+                for a, a0 in enumerate(a_list):
+                    Na = a0*I0
+                    for m, maxDet in enumerate(m_list):
+                        E = [self.evalImgs[Nk + Na + i] for i in i_list]
+                        E = [e for e in E if not e is None]
+                        if len(E) == 0:
+                            continue
+                        dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E])
+
+                        # different sorting method generates slightly different results.
+                        # mergesort is used to be consistent as Matlab implementation.
+                        inds = np.argsort(-dtScores, kind='mergesort')
+                        dtScoresSorted = dtScores[inds]
+
+                        dtm  = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds]
+                        dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet]  for e in E], axis=1)[:,inds]
+                        gtIg = np.concatenate([e['gtIgnore'] for e in E])
+                        npig = np.count_nonzero(gtIg==0 )
+                        if npig == 0:
+                            continue
+                        tps = np.logical_and(               dtm,  np.logical_not(dtIg) )
+                        fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) )
+
+                        tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float)
+                        fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float)
+                        for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):
+                            tp = np.array(tp)
+                            fp = np.array(fp)
+                            nd = len(tp)
+                            rc = tp / npig
+                            pr = tp / (fp+tp+np.spacing(1))
+                            q  = np.zeros((R,))
+                            ss = np.zeros((R,))
+
+                            if nd:
+                                recall[t,k,a,m] = rc[-1]
+                            else:
+                                recall[t,k,a,m] = 0
+
+                            # numpy is slow without cython optimization for accessing elements
+                            # use python array gets significant speed improvement
+                            pr = pr.tolist(); q = q.tolist()
+
+                            for i in range(nd-1, 0, -1):
+                                if pr[i] > pr[i-1]:
+                                    pr[i-1] = pr[i]
+
+                            inds = np.searchsorted(rc, p.recThrs, side='left')
+                            try:
+                                for ri, pi in enumerate(inds):
+                                    q[ri] = pr[pi]
+                                    ss[ri] = dtScoresSorted[pi]
+                            except:
+                                pass
+                            precision[t,:,k,a,m] = np.array(q)
+                            scores[t,:,k,a,m] = np.array(ss)
+            self.eval = {
+                'params': p,
+                'counts': [T, R, K, A, M],
+                'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+                'precision': precision,
+                'recall':   recall,
+                'scores': scores,
+            }
+
+        toc = time.time()
+        print('DONE (t={:0.2f}s).'.format( toc-tic))
+
+    def summarize(self):
+        '''
+        Compute and display summary metrics for evaluation results.
+        Note this functin can *only* be applied on the default parameter setting
+        '''
+        def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100, part=None):
+            p = self.params
+            iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
+            titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
+            typeStr = '(AP)' if ap==1 else '(AR)'
+            iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \
+                if iouThr is None else '{:0.2f}'.format(iouThr)
+
+            aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]
+            mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
+            if ap == 1:
+                # dimension of precision: [TxRxKxAxM]
+                if part:
+                    s = self.eval[part]['precision']
+                else:
+                    s = self.eval['precision']
+                # IoU
+                if iouThr is not None:
+                    t = np.where(iouThr == p.iouThrs)[0]
+                    s = s[t]
+                s = s[:,:,:,aind,mind]
+            else:
+                # dimension of recall: [TxKxAxM]
+                if part:
+                    s = self.eval[part]['recall']
+                else:
+                    s = self.eval['recall']
+                if iouThr is not None:
+                    t = np.where(iouThr == p.iouThrs)[0]
+                    s = s[t]
+                s = s[:,:,aind,mind]
+            if len(s[s>-1])==0:
+                mean_s = -1
+            else:
+                mean_s = np.mean(s[s>-1])
+            print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))
+            return mean_s
+        def _summarizeDets():
+            stats = np.zeros((12,))
+            stats[0] = _summarize(1)
+            stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2])
+            stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2])
+            stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2])
+            stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2])
+            stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2])
+            stats[6] = _summarize(0, maxDets=self.params.maxDets[0])
+            stats[7] = _summarize(0, maxDets=self.params.maxDets[1])
+            stats[8] = _summarize(0, maxDets=self.params.maxDets[2])
+            stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2])
+            stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2])
+            stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2])
+            return stats
+        def _summarizeKps(part='fullbody'):
+            stats = np.zeros((10,))
+            stats[0] = _summarize(1, maxDets=20, part=part)
+            stats[1] = _summarize(1, maxDets=20, iouThr=.5, part=part)
+            stats[2] = _summarize(1, maxDets=20, iouThr=.75, part=part)
+            stats[3] = _summarize(1, maxDets=20, areaRng='medium', part=part)
+            stats[4] = _summarize(1, maxDets=20, areaRng='large', part=part)
+            stats[5] = _summarize(0, maxDets=20, part=part)
+            stats[6] = _summarize(0, maxDets=20, iouThr=.5, part=part)
+            stats[7] = _summarize(0, maxDets=20, iouThr=.75, part=part)
+            stats[8] = _summarize(0, maxDets=20, areaRng='medium', part=part)
+            stats[9] = _summarize(0, maxDets=20, areaRng='large', part=part)
+            return {part: stats}
+        if not self.eval:
+            raise Exception('Please run accumulate() first')
+        iouType = self.params.iouType
+        if iouType == 'segm' or iouType == 'bbox':
+            summarize = _summarizeDets
+            self.stats = summarize()
+        elif iouType == 'keypoints':
+            summarize = _summarizeKps
+            parts = ['body', 'foot', 'face', 'hand', 'fullbody']
+            for part in parts:
+                print(part, ':')
+                self.stats.append(summarize(part))
+
+    def __str__(self):
+        self.summarize()
+
+class Params:
+    '''
+    Params for coco evaluation api
+    '''
+    def setDetParams(self):
+        self.imgIds = []
+        self.catIds = []
+        # np.arange causes trouble.  the data point on arange is slightly larger than the true value
+        self.iouThrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05) + 1), endpoint=True)
+        self.recThrs = np.linspace(.0, 1.00, int(np.round((1.00 - .0) / .01) + 1), endpoint=True)
+        self.maxDets = [1, 10, 100]
+        self.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, 32 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]]
+        self.areaRngLbl = ['all', 'small', 'medium', 'large']
+        self.useCats = 1
+
+    def setKpParams(self):
+        self.imgIds = []
+        self.catIds = []
+        # np.arange causes trouble.  the data point on arange is slightly larger than the true value
+        self.iouThrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05) + 1), endpoint=True)
+        self.recThrs = np.linspace(.0, 1.00, int(np.round((1.00 - .0) / .01) + 1), endpoint=True)
+        self.maxDets = [20]
+        self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]]
+        self.areaRngLbl = ['all', 'medium', 'large']
+        self.useCats = 1
+
+    def __init__(self, iouType='segm'):
+        if iouType == 'segm' or iouType == 'bbox':
+            self.setDetParams()
+        elif iouType == 'keypoints':
+            self.setKpParams()
+        else:
+            raise Exception('iouType not supported')
+        self.iouType = iouType
+        # useSegm is deprecated
+        self.useSegm = None