python-wml 3.0.0__py3-none-any.whl

wml/iotoolkit/common.py

@@ -0,0 +1,108 @@
+import os
+import wml.wml_utils as wmlu
+import PIL
+import wml.img_utils as wmli
+import os.path as osp
+import glob
+from collections import namedtuple
+
+DetData = namedtuple('DetData','path,img_shape,labels,labels_name,bboxes,masks,area,is_crowd,extra_data') #img_shape:(H,W)
+DetBboxesData = namedtuple('DetBboxesdata','path,img_shape,labels,bboxes,is_crowd')
+
+def __get_resample_nr(labels,resample_parameters):
+    nr = 1
+    for l in labels:
+        if l in resample_parameters:
+            nr = max(nr,resample_parameters[l])
+    return nr
+
+def resample(files,labels,resample_parameters):
+    '''
+    files: list of files
+    labels: list of labels
+    resample_parameters: {labels:resample_nr}
+    '''
+    new_files = []
+    repeat_files_nr = 0
+    repeat_nr = 0
+    for f,l in zip(files,labels):
+        nr = __get_resample_nr(l,resample_parameters)
+        if nr>1:
+            new_files = new_files+[f]*nr
+            #print(f"Repeat {f} {nr} times.")
+            repeat_files_nr += 1
+            repeat_nr += nr
+        elif nr==1:
+            new_files.append(f)
+    print(f"Total repeat {repeat_files_nr} files, total repeat {repeat_nr} times.")
+    print(f"{len(files)} old files --> {len(new_files)} new files")
+
+    return new_files
+
+def get_shape_from_img(xml_path,img_path):
+    '''
+    return: [H,W]
+    '''
+    if not os.path.exists(img_path):
+        img_path = wmlu.change_suffix(xml_path, "jpg")
+        if not os.path.exists(img_path):
+            img_path = wmlu.change_suffix(xml_path, "jpeg")
+    return wmli.get_img_size(img_path)
+
+def ignore_case_dict_label_text2id(name,dict_data):
+    name = name.lower()
+    if name not in dict_data:
+        print(f"ERROR: trans {name} faild.")
+    return dict_data.get(name,None)
+
+def dict_label_text2id(name,dict_data):
+    if name not in dict_data:
+        print(f"ERROR: trans {name} faild.")
+    return dict_data.get(name,None)
+
+def find_imgs_for_ann_file(ann_path):
+    ann_path = osp.abspath(ann_path)
+    img_suffix = [".jpg",".jpeg",".bmp",".png",".gif"]
+    pattern = wmlu.change_suffix(ann_path,"*")
+    files = glob.glob(pattern)
+    img_file = None
+    for file in files:
+        if file==ann_path:
+            continue
+        if osp.splitext(file)[1].lower() in img_suffix:
+            img_file = file
+        else:
+            print(f"WARNING: Unknow img format file {file} for {ann_path}")
+            img_file = file
+    return img_file
+
+
+def get_auto_dataset_suffix(data_dir,suffix="auto"):
+
+    if isinstance(data_dir,(list,tuple)):
+        data_dir = data_dir[0]
+    if suffix.lower() != "auto":
+        return suffix
+
+    if isinstance(data_dir,str) and osp.isfile(data_dir):
+        with open(data_dir,"r") as f:
+            data_dir = f.readline().strip()
+        data_dir = data_dir.split(",")[0]
+        data_dir = osp.dirname(data_dir)
+
+    for f in wmlu.find_files(data_dir,suffix=".json"):
+        return "json"
+
+    for f in wmlu.find_files(data_dir,suffix=".xml"):
+        return "xml"
+
+    return "none"
+
+def check_dataset_dir(dir_path):
+    if isinstance(dir_path,(list,tuple)):
+        return dir_path
+    if "," in dir_path:
+        dir_path = dir_path.split(",")
+        return [osp.abspath(osp.expanduser(p)) for p in dir_path]
+    else:
+        return osp.abspath(osp.expanduser(dir_path))

wml/iotoolkit/crowd_pose.py

@@ -0,0 +1,146 @@
+import json
+import numpy as np
+import wml.object_detection2.visualization as odv
+import wml.object_detection2.bboxes as odb
+import os.path as osp
+import wml.wml_utils as wmlu
+import wml.img_utils as wmli
+import wml.object_detection2.keypoints as odk
+import math
+
+'''
+'left_shoulder', 'right_shoulder', 0,1
+'left_elbow', 'right_elbow', 2,3
+ 'left_wrist', 'right_wrist', 4,5
+'left_hip', 'right_hip', 6,7
+ 'left_knee', 'right_knee', 8,9
+ 'left_ankle', 'right_ankle', 10,11
+ 'head', 'neck' 12,13
+'''
+def read_crowd_pose(file_path):
+    with open(file_path,"r") as f:
+        datas = json.load(f)
+    id2img = {}
+    for data in datas['images']:
+        id = data['id']
+        file_name = data['file_name']
+        id2img[id] = file_name
+
+    res = {}
+    for data in datas['annotations']:
+        kps = data['keypoints']
+        kps = np.array(kps,dtype=np.float32)
+        kps = np.reshape(kps,[-1,3])
+        if data['iscrowd']:
+            continue
+        mask = kps[...,2]
+        if np.sum(mask)<2:
+            continue
+        if np.count_nonzero(mask>0.1)<2:
+            continue
+        bbox = data['bbox']
+        bbox = [bbox[0],bbox[1],bbox[0]+bbox[2],bbox[1]+bbox[3]]
+        kps_bbox = odk.npget_bbox(kps)
+        bbox = odb.bbox_of_boxes([bbox,kps_bbox])
+        id = data['image_id']
+        image = id2img[id]
+        if id not in res:
+            res[id] = [image,[kps],[bbox]]
+        else:
+            res[id][1].append(kps)
+            res[id][2].append(bbox)
+    
+    r_res = []
+    for k,v in res.items():
+        r_res.append([v[0],np.array(v[1]),np.array(v[2])])
+
+    return r_res
+
+class Trans2COCO:
+    def __init__(self) -> None:
+        self.dst_idxs = [5,6,7,8,9,10,11,12,13,14,15,16]
+        self.src_idxs = np.array([0,1,2,3,4,5,6,7,8,9,10,11],dtype=np.int32)
+        self.coco_idxs = [0,1,2,3,4]
+    
+    @staticmethod
+    def get_head_pos(kp):
+        p0 = kp[12]
+        p1 = kp[13]
+        if p0[2]>0 and p1[2]>0:
+            dx = p0[0] - p1[0]
+            dy = p0[1] - p1[1]
+            dis = math.sqrt(dx*dx+dy*dy)*0.75
+            cx = (p0[0] + p1[0])/2
+            cy = (p0[1] + p1[1])/2
+            x0 = cx-dis
+            x1 = cx+dis
+            y0 = cy-dis
+            y1 = cy+dis
+            return [x0,y0,x1,y1]
+        else:
+            return None
+    
+    @staticmethod 
+    def kps_in_bbox(kps,bbox):
+        nr = kps.shape[0]
+        for i in range(nr):
+            if kps[i,2]<= 0:
+                return False
+            x = kps[i,0]
+            y = kps[i,1]
+            if x<bbox[0] or x>bbox[2] or y<bbox[1] or y>bbox[3]:
+                return False
+        return True
+
+
+    def __call__(self,mpii_kps,coco_kps):
+        if len(mpii_kps.shape)==2:
+            return self.trans_one(mpii_kps,coco_kps)
+        res = []
+        for mp,coco in zip(mpii_kps,coco_kps):
+            res.append(self.trans_one(mp,coco))
+        return np.array(res)
+
+    '''def trans_one(self,mpii_kps,coco_kps):
+        res = np.zeros([17,3],dtype=np.float32)
+        res[self.dst_idxs] = mpii_kps[self.src_idxs]
+        res[self.coco_idxs] = coco_kps[self.coco_idxs]
+        return res'''
+    
+    def trans_one(self,mpii_kps,coco_kps=None):
+        '''
+        img: [RGB]
+        '''
+        res = np.zeros([17,3],dtype=np.float32)
+        res[self.dst_idxs] = mpii_kps[self.src_idxs]
+        if coco_kps is not None:
+            left_right_pairs = [[5,6],[11,12]]
+            is_good = True
+            for pair in left_right_pairs:
+                l,r = pair
+                if res[l,0]<res[r,0] or res[l,2]<0.1 or res[r,2]<0.1:
+                    is_good = False
+                    break
+
+            head_bbox = self.get_head_pos(mpii_kps)
+            if is_good and head_bbox is not None and self.kps_in_bbox(coco_kps[self.coco_idxs],head_bbox):
+                res[self.coco_idxs] = coco_kps[self.coco_idxs]
+        return res
+
+if __name__ == "__main__":
+    file_path = '/home/wj/ai/mldata1/crowd_pose/CrowdPose/crowdpose_train.json'
+    images_dir = '/home/wj/ai/mldata1/crowd_pose/images'
+    save_dir = '/home/wj/ai/mldata1/crowd_pose/tmp/vis'
+    wmlu.create_empty_dir(save_dir,remove_if_exists=False)
+    datas = read_crowd_pose(file_path)
+    do_vis = True
+    for data in datas:
+        image_name,kps,bbox = data
+        image = osp.join(images_dir,image_name)
+        img = wmli.imread(image)
+        img = odv.draw_keypoints(img, kps, no_line=True)
+        t_bboxes = np.array([bbox])
+        t_bboxes = odb.npchangexyorder(t_bboxes)
+        img = odv.draw_bboxes(img, bboxes=t_bboxes, is_relative_coordinate=False)
+        save_path = osp.join(save_dir,image_name)
+        wmli.imwrite(save_path, img)

wml/iotoolkit/fast_labelme.py

@@ -0,0 +1,110 @@
+import wml.wml_utils as wmlu
+import os
+import json
+import numpy as np
+import cv2 as cv
+import copy
+import wml.img_utils as wmli
+import random
+import matplotlib.pyplot as plt
+import sys
+import cv2
+from wml.object_detection2.standard_names import *
+import wml.object_detection2.bboxes as odb
+from functools import partial
+from .common import *
+from .labelme_toolkit_fwd import *
+from wml.semantic.structures import *
+from .labelme_base import LabelMeBase
+import glob
+
+class FastLabelMeData(LabelMeBase):
+    '''
+    与LabelMeData的区别为生成的Mask使用多边形的方式保存
+    '''
+    def __init__(self,label_text2id=None,shuffle=True,absolute_coord=True,
+                 filter_empty_files=False,
+                 resample_parameters=None,
+                 read_data_kwargs={'circle_points_nr':20},**kwargs):
+        '''
+        label_text2id: func(name)->int
+        '''
+        super().__init__(label_text2id=label_text2id,
+                         shuffle=shuffle,
+                         absolute_coord=absolute_coord,
+                         filter_empty_files=filter_empty_files,
+                         resample_parameters=resample_parameters,
+                         read_data_kwargs=read_data_kwargs,**kwargs)
+        
+    def __getitem__(self,idx):
+        '''
+        :return: 
+        full_path,img_size,category_ids,category_names,boxes,binary_masks,area,is_crowd,num_annotations_skipped
+        binary_masks:[N,H,W]
+        bboxes:[N,4] (ymin,xmin,ymax,xmax)
+        '''
+        img_file, json_file = self.files[idx]
+        image, annotations_list = read_labelme_data(json_file, None,mask_on=self.mask_on,use_semantic=True,use_polygon_mask=True,
+                                                    **self.read_data_kwargs)
+        labels_names,bboxes = get_labels_and_bboxes(image,annotations_list,is_relative_coordinate=not self.absolute_coord)
+        masks = [ann["segmentation"] for ann in annotations_list] if self.mask_on else None
+        difficult = np.array([v['difficult'] for v in annotations_list],dtype=np.bool)
+        img_height = image['height']
+        img_width = image['width']
+        if masks is not None:
+            masks = WPolygonMasks(masks,width=img_width,height=img_height)
+        
+        if self.label_text2id is not None:
+            try:
+                labels = [self.label_text2id(x) for x in labels_names]
+            except:
+                labels = []
+            keep = [x is not None for x in labels]
+            labels = [x if x is not None else -1 for x in labels]
+            labels = np.array(labels,dtype=np.int32)
+            labels = labels[keep]
+            bboxes = bboxes[keep]
+            if masks is not None:
+                masks = masks[keep]
+            difficult = difficult[keep]
+            labels_names = np.array(labels_names)[keep]
+        else:
+            labels = None
+            
+        return DetData(img_file, [image['height'],image['width']],labels, labels_names, bboxes, masks, None, difficult,None)
+    
+            
+if __name__ == "__main__":
+    #data_statistics("/home/vghost/ai/mldata/qualitycontrol/rdatasv3")
+    import wml.img_utils as wmli
+    import wml.object_detection2.visualization as odv
+    import matplotlib.pyplot as plt
+    ID_TO_TEXT = {1:{"id":1,"name":"a"},2:{"id":2,"name":"b"},3:{"id":3,"name":"c"}}
+    NAME_TO_ID = {}
+    for k,v in ID_TO_TEXT.items():
+        NAME_TO_ID[v["name"]] = v["id"]
+    def name_to_id(name):
+        return NAME_TO_ID[name]
+
+    data = FastLabelMeData(label_text2id=name_to_id,shuffle=True)
+    #data.read_data("/data/mldata/qualitycontrol/rdatasv5_splited/rdatasv5")
+    #data.read_data("/home/vghost/ai/mldata2/qualitycontrol/rdatav10_preproc")
+    #data.read_data("/home/vghost/ai/mldata2/qualitycontrol/rdatasv10_neg_preproc")
+    data.read_data("/home/vghost/ai/mldata2/qualitycontrol/rdatasv10_1x_neg_preproc")
+    #data.read_data("/home/vghost/ai/mldata2/qualitycontrol/x")
+    for x in data.get_items():
+        full_path, img_info,category_ids, category_names, boxes, binary_mask, area, is_crowd, num_annotations_skipped = x
+        img = wmli.imread(full_path)
+
+
+        def text_fn(classes, scores):
+            return ID_TO_TEXT[classes]['name']
+
+        odv.draw_bboxes_and_maskv2(
+            img=img, classes=category_ids, scores=None, bboxes=boxes, masks=binary_mask, color_fn=None,
+            text_fn=text_fn, thickness=4,
+            show_text=True,
+            fontScale=0.8)
+        plt.figure()
+        plt.imshow(img)
+        plt.show()

wml/iotoolkit/image_folder.py

@@ -0,0 +1,95 @@
+import wml.wml_utils as wmlu
+import os.path as osp
+import wml.img_utils as wmli
+import copy
+
+class ImageFolder:
+    def __init__(self,label_text2id=None,classes=None):
+        self.label_text2id = label_text2id
+        self._classes = classes
+        self._data = []
+        pass
+
+    def read_one_dir(self,data_dir):
+        sub_dirs = wmlu.get_subdir_in_dir(data_dir)
+        if self._classes is None:
+            self._classes = copy.deepcopy(sub_dirs)
+        else:
+            for sd in sub_dirs:
+                if sd not in self._classes:
+                    self._classes.append(sd)
+        for dir in sub_dirs:
+            label = dir.lower()
+            if self.label_text2id is not None:
+                label = self.label_text2id(label)
+            files = wmlu.get_files(osp.join(data_dir,dir),suffix=wmli.BASE_IMG_SUFFIX)
+            for f in files:
+                self._data.append((label,f))
+
+    def read_data(self,data_dir):
+        self._data = []
+        if isinstance(data_dir,(str,bytes)):
+            self.read_one_dir(data_dir)
+        elif isinstance(data_dir,(list,tuple)):
+            for dir in data_dir:
+                self.read_one_dir(dir)
+        else:
+            print(f"ERROR: error data dir type {type(data_dir)},  {data_dir}")
+
+    @property
+    def classes(self):
+        return self._classes
+            
+    def __len__(self):
+        return len(self._data)
+
+    def __getitem__(self,idx):
+        return self._data[idx]
+
+
+    @staticmethod
+    def get_label(file_path):
+        dirname = osp.dirname(osp.abspath(file_path))
+        label = osp.basename(dirname).lower()
+        if "(" in label and ")" in label: 
+            idx0 = label.index("(")
+            idx1 = label.index(")")
+            if idx0<idx1:
+                label = label[:idx0]+label[idx1+1:]
+        return label
+
+class ImageFolder2:
+    '''
+    使用文件父目录作为标签
+    '''
+    def __init__(self,label_text2id=None,classes=None):
+        self.label_text2id = label_text2id
+        self._classes = classes
+        self._data = []
+        pass
+
+    def read_data(self,data_dir):
+        files = wmlu.get_files(data_dir,suffix=wmli.BASE_IMG_SUFFIX)
+        classes = set()
+        for f in files:
+            label = ImageFolder2.get_label(f)
+            self._data.append((label,f))
+            classes.add(label)
+        if self._classes is None:
+            self._classes = list(classes)
+
+    @property
+    def classes(self):
+        return self._classes
+            
+    def __len__(self):
+        return len(self._data)
+
+    def __getitem__(self,idx):
+        return self._data[idx]
+
+
+    @staticmethod
+    def get_label(file_path):
+        dirname = osp.dirname(osp.abspath(file_path)).replace(" ","")
+        return osp.basename(dirname).lower()

wml/iotoolkit/imgs_cache.py

@@ -0,0 +1,58 @@
+import numpy as np
+import cv2
+
+class ImgsCache(object):
+    def __init__(self,files,img_size=None,cache_limit=-1,mem_limit=-1):
+        '''
+        Args:
+            files: list of img files path.
+            img_size: (H,W)
+            cache_limit: cache number limit
+            mem_limit: meme limit (G)
+        '''
+        self.img_files = files
+        self.img_size = img_size
+        if cache_limit<=1 and mem_limit>0 and img_size is not None:
+            per_img_size = img_size[0]*img_size[1]*3
+            cache_limit = mem_limit*1e6/per_img_size
+        self.cache_limit = cache_limit
+        self.cache_data = {}
+        print(f"Cache limit is {self.cache_limit}.")
+
+    def __getitem__(self, item):
+        assert isinstance(item,int), f"Error item type: {item}"
+        if item in self.cache_data:
+            return self.cache_data[item].copy()
+        else:
+            img = self.load_image_data(item)
+            if self.cache_limit<=1 \
+                    or (self.cache_limit>1 and len(self.cache_data)<self.cache_limit):
+                self.cache_data[item] = img.copy()
+            return img
+
+    def load_image_data(self,idx):
+        if self.img_size is not None:
+            return self.load_resized_img(idx)
+        else:
+            return self.load_image(idx)
+
+    def load_resized_img(self, index):
+        img = self.load_image(index)
+        r = min(self.img_size[0] / img.shape[0], self.img_size[1] / img.shape[1])
+        resized_img = cv2.resize(
+            img,
+            (int(img.shape[1] * r), int(img.shape[0] * r)),
+            interpolation=cv2.INTER_LINEAR,
+        ).astype(np.uint8)
+
+        return resized_img
+
+    def load_image(self, index):
+        img_file = self.img_files[index]
+        img = cv2.imread(img_file, cv2.IMREAD_COLOR)
+        if img is None or img.shape[0]<2 or img.shape[1]<2:
+            print(f"ERROR: error img {img}, shape {img.shape if img is not None else 0}, file_path {self.img_files[index]}")
+            raise Exception("Empty img.")
+
+        return img
+

wml/iotoolkit/imgs_reader_mt.py

@@ -0,0 +1,73 @@
+import wml.img_utils as wmli
+import wml.wml_utils as wmlu
+from wml.wtorch.data.dataloader import DataLoader
+from wml.wtorch.data._utils.collate import null_convert,default_collate
+import numpy as np
+import random
+import sys
+import os
+from PIL import Image
+Image.MAX_IMAGE_PIXELS = 1000000000 
+
+class MaxImgLongSize:
+    def __init__(self,max_size=2048):
+        self.max_size = max_size
+    
+    def __call__(self,img):
+        if img.shape[0]>self.max_size or img.shape[1]>self.max_size:
+            img = wmli.resize_long_size(img,self.max_size)
+        return img
+
+    def __repr__(self):
+        return f"{type(self).__name__}: max_size={self.max_size}"
+
+class ImgsDataset:
+    def __init__(self,data_dir_or_files,transform=None,shuffle=True):
+        if isinstance(data_dir_or_files,str):
+            self.files = wmlu.get_files(data_dir_or_files,suffix=wmli.BASE_IMG_SUFFIX)
+        else:
+            self.files = data_dir_or_files
+        if shuffle:
+            random.shuffle(self.files)
+        self.transform = transform
+        print(f"ImgsDataset transform:")
+        print(self.transform)
+
+    def __len__(self):
+        return len(self.files)
+
+    def __getitem__(self, item):
+        path = self.files[item]
+        try:
+            sys.stdout.flush()
+            img = wmli.hpimread(path)
+            if self.transform is not None:
+                img = self.transform(img)
+            return path,img
+            #return path,wmli.gpu_imread(path)
+        except Exception as e:
+            print(f"ERROR: {e}")
+            sys.stdout.flush()
+            return path,np.zeros([0,0,1],dtype=np.uint8)
+
+class ImgsReader:
+    def __init__(self, data_dir_or_files, thread_nr=8,transform=None,shuffle=True):
+        self.dataset = ImgsDataset(data_dir_or_files,transform=transform,shuffle=shuffle)
+
+        dataloader_kwargs = {"num_workers": thread_nr, "pin_memory": False}
+        dataloader_kwargs["sampler"] = list(range(len(self.dataset)))
+        dataloader_kwargs["batch_size"] = None
+        dataloader_kwargs["batch_split_nr"] = 1
+        dataloader_kwargs['collate_fn'] = null_convert
+        #dataloader_kwargs['collate_fn'] = default_collate
+
+        data_loader = DataLoader(self.dataset, **dataloader_kwargs)
+
+        self.data_loader = data_loader
+        self.data_loader_iter = iter(self.data_loader)
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __iter__(self):
+        return self.data_loader_iter

wml/iotoolkit/labelme_base.py

@@ -0,0 +1,102 @@
+import wml.wml_utils as wmlu
+import os
+import json
+import numpy as np
+import cv2 as cv
+import copy
+import wml.img_utils as wmli
+import random
+import matplotlib.pyplot as plt
+import sys
+import cv2
+from wml.object_detection2.standard_names import *
+import wml.object_detection2.bboxes as odb
+from functools import partial
+from .common import *
+from .labelme_toolkit_fwd import *
+import glob
+from .base_dataset import BaseDataset
+
+class LabelMeBase(BaseDataset):
+    def __init__(self,label_text2id=None,shuffle=True,absolute_coord=True,
+                 filter_empty_files=False,
+                 filter_error=False,
+                 resample_parameters=None,
+                 use_polygon_mask=False,
+                 silent=False,
+                 keep_no_ann_imgs=False,
+                 mask_on=True,
+                 read_data_kwargs={'circle_points_nr':20}):
+        '''
+        label_text2id: func(name)->int
+        '''
+        self.files = None
+        super().__init__(label_text2id=label_text2id,
+                          filter_empty_files=filter_empty_files,
+                          filter_error=filter_error,
+                          resample_parameters=resample_parameters,
+                          shuffle=shuffle,
+                          silent=silent,
+                          absolute_coord=absolute_coord,
+                          keep_no_ann_imgs=keep_no_ann_imgs,
+                          mask_on=mask_on)
+        self.read_data_kwargs = read_data_kwargs
+        self.use_polygon_mask = use_polygon_mask
+
+    def find_files_in_dir(self,dir_path,img_suffix=".jpg;;.bmp;;.png;;.jpeg"):
+        files = get_files(dir_path,img_suffix=img_suffix,keep_no_json_img=self.keep_no_ann_imgs)
+        return files
+
+    def get_labels(self,fs):
+        img_file,json_file = fs
+        image, annotations_list = read_labelme_data(json_file, None,use_semantic=True,mask_on=False,
+                                                    use_polygon_mask=True,
+                                                    do_raise=True,
+                                                    **self.read_data_kwargs)
+        labels_names,bboxes = get_labels_and_bboxes(image,annotations_list,is_relative_coordinate=not self.absolute_coord)
+        if self.label_text2id:
+            labels = [self.label_text2id(x) for x in labels_names] #测试转label是否有误
+        else:
+            labels = None
+        
+        return labels,labels_names
+    
+    def get_ann_info(self,idx):
+        img_file,json_file = self.files[idx]
+        with open(json_file,"r",encoding="gb18030") as f:
+            data_str = f.read()
+            image = {}
+            try:
+                json_data = json.loads(data_str)
+                img_width = int(json_data["imageWidth"])
+                img_height = int(json_data["imageHeight"])
+                image["height"] = int(img_height)
+                image["width"] = int(img_width)
+                image["file_name"] = wmlu.base_name(json_file)
+            except:
+                pass
+        
+        return image
+    
+    def get_items(self):
+        '''
+        :return: 
+        full_path,img_size,category_ids,category_names,boxes,binary_masks,area,is_crowd,num_annotations_skipped
+        '''
+        for i in range(len(self.files)):
+            sys.stdout.write('\r>> read data %d/%d' % (i + 1, len(self.files)))
+            sys.stdout.flush()
+            yield self.__getitem__(i)
+
+    def get_boxes_items(self):
+        '''
+        :return: 
+        full_path,img_size,category_ids,boxes,is_crowd
+        '''
+        for i,(img_file, json_file) in enumerate(self.files):
+            sys.stdout.write('\r>> read data %d/%d' % (i + 1, len(self.files)))
+            sys.stdout.flush()
+            image, annotations_list = read_labelme_data(json_file, None,mask_on=False,**self.read_data_kwargs)
+            labels_names,bboxes = get_labels_and_bboxes(image,annotations_list,is_relative_coordinate=not self.absolute_coord)
+            labels = [self.label_text2id(x) for x in labels_names]
+            yield DetBboxesData(img_file,[image['height'],image['width']],labels, bboxes,  None)