pybaseutils 2.2.22__tar.gz → 2.2.25__tar.gz

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: pybaseutils
-Version: 2.2.22
+Version: 2.2.25
 Summary: pybaseutils
 Home-page: https://github.com/PanJinquan/base-utils
 Author: PanJinquan
@@ -41,9 +41,9 @@ pybaseutils是个人开发常用的python库，集成了python等常用的算法
    (有延时，可能不是最新版本)
 -  安装方法2：pip install –upgrade -i https://pypi.org/simple
    pybaseutils (从pypi源下载最新版本)
-
-(设置默认的镜像源：pip config set global.index-url
-https://pypi.tuna.tsinghua.edu.cn/simple)
+-  设置默认的镜像源：pip config set global.index-url
+   https://pypi.tuna.tsinghua.edu.cn/simple)
+-  超时异常：pip –default-timeout=1000000000
 
 一些问题修复说明
 ----------------

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/README.md

@@ -20,8 +20,8 @@ pybaseutils是个人开发常用的python库，集成了python等常用的算法
 
 - 安装方法1：pip install --upgrade -i https://pypi.tuna.tsinghua.edu.cn/simple pybaseutils (有延时，可能不是最新版本)
 - 安装方法2：pip install --upgrade -i https://pypi.org/simple pybaseutils (从pypi源下载最新版本)
-
-(设置默认的镜像源：pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple)
+- 设置默认的镜像源：pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple)
+- 超时异常：pip --default-timeout=1000000000 
 
 ## 一些问题修复说明
 

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/__init__.py

@@ -8,4 +8,4 @@
 # --------------------------------------------------------
 """
 
-__version__ = '2.2.22'
+__version__ = '2.2.25'

pybaseutils-2.2.25/pybaseutils/converter/prelabelme.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+# --------------------------------------------------------
+# @Author : Pan
+# @E-mail : 
+# @Date   : 2025-04-29 09:13:09
+# @Brief  :
+# --------------------------------------------------------
+"""
+import os
+import cv2
+from tqdm import tqdm
+from pybaseutils.converter import build_labelme
+from pybaseutils import http_utils, base64_utils, file_utils, image_utils
+
+
+class PreLabelme(object):
+    """
+    调用API接口进行预标注，并保持为Labelme数据格式
+    接口必须返回字段:
+            boxes 必选
+            label 必选
+            score 可选
+            names 可选
+            segms 可选
+    """
+
+    def __init__(self, url, headers=None, timeout=6, max_retries=1):
+        """
+        :param url:
+        """
+        self.url = url
+        self.headers = headers
+        self.timeout = timeout
+        self.max_retries = max_retries
+
+    def request(self, params, **kwargs):
+        """
+        :param params:
+        :return: boxes,label,names,segms,,score
+        """
+        params.update(kwargs)
+        params = base64_utils.serialization(params)
+        result = http_utils.post(self.url, params=params, headers=self.headers,
+                                 timeout=self.timeout, max_retries=self.max_retries)
+        result = base64_utils.deserialization(result)
+        return result
+
+    def draw_result(self, image, result: dict, vis=True, delay=0):
+        """
+        :param image: BGR image
+        :param boxes:<np.ndarray>: (num_boxes, 4), box=[xmin,ymin,xmax,ymax]
+        :param score:<np.ndarray>: (num_boxes,)
+        :param label:<np.ndarray>: (num_boxes,)
+        :return:
+        """
+        color = (255, 0, 0)
+        boxes = result.get("boxes", [])
+        score = result.get("score", [])
+        landm = result.get("landm", [])
+        segms = result.get("segms", [])
+        label = result.get("label", [])
+        names = result.get("names", [])
+        if not names: names = label
+        texts = ["{} {:3.3f}".format(n, s) for n, s in zip(names, score)]
+        image = image_utils.draw_image_bboxes_text(image, boxes, texts, color=color, thickness=2,
+                                                   fontScale=0.8, drawType="ch")
+        image = image_utils.draw_landmark(image, landm, radius=2, fontScale=1.0, color=color)
+        if vis: image_utils.cv_show_image("image", image, delay=delay)
+        return image
+
+    def detect_image_dir(self, image_dir, vis=True):
+        """
+        :param image_dir: directory or image file path
+        :param vis:<bool>
+        :return:
+        """
+        image_list = file_utils.get_files_lists(image_dir)
+        for image_file in tqdm(image_list):
+            image = cv2.imread(image_file)
+            result = self.request(params={"image": image})
+            if vis: image = self.draw_result(image, result)
+
+    def prelabel_labelme(self, image_dir, clsss_dict={}, vis=True):
+        """
+        进行预标注,并输出labelme数据格式
+        :param image_dir: directory or image file path
+        :param vis:<bool>
+        :return:
+        """
+        image_list = file_utils.get_files_lists(image_dir)
+        for image_file in tqdm(image_list):
+            image = cv2.imread(image_file)
+            if image is None: continue
+            h, w = image.shape[:2]
+            result = self.request(params={"image": image})
+            image_name = os.path.basename(image_file)
+            json_file = file_utils.change_postfix(image_file, postfix=".json")
+            boxes = result.get("boxes", [])
+            segms = result.get("segms", [])
+            label = result.get("label", [])
+            names = [clsss_dict[l] for l in label] if clsss_dict else result.get("names", [])
+            result['names'] = names
+            points = image_utils.boxes2polygons(boxes)
+            build_labelme.maker_labelme(json_file, points, names, image_name, image_size=(w, h), image_bs64=None)
+            if vis: image = self.draw_result(image, result)
+
+
+if __name__ == '__main__':
+    # url = 'http://192.168.68.102:5000/detect'
+    url = 'http://0.0.0.0:5000/detect'
+    image_dir = "/media/PKing/新加卷1/个人文件/video/video/frame"
+    s = PreLabelme(url=url)
+    s.prelabel_labelme(image_dir=image_dir, vis=False)
+    # s.detect_image_dir(image_dir=image_dir)

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/coords_utils.py

@@ -23,6 +23,7 @@ def clip_xyxy(xyxy: np.ndarray, valid_range):
     :return:
     """
     if len(xyxy) == 0: return xyxy
+    if not isinstance(xyxy, np.ndarray): xyxy = np.asarray(xyxy)
     xmin, ymin, xmax, ymax = valid_range
     xyxy[:, [0, 2]] = np.clip(xyxy[:, [0, 2]], xmin, xmax)
     xyxy[:, [1, 3]] = np.clip(xyxy[:, [1, 3]], ymin, ymax)
@@ -105,7 +106,7 @@ def cxcywh2xyxy(cxcywh: np.ndarray, width=None, height=None, normalized=False):
 def extend_xyxy(xyxy: np.ndarray, scale=[1.0, 1.0], valid_range=[], fixed=False, use_max=True):
     """
     :param bboxes: [[xmin, ymin, xmax, ymax]]
-    :param scale: [sx,sy]==>(W,H)
+    :param scale: [sx,sy]==>(W,H),（sx1,sy1,sx2,sy2）,1.0表示不缩放，<1.0表示缩小倍数，>1.0表示扩大倍数
     :param valid_range:有效范围(xmin,ymin,xmax,ymax)
     :param fixed: 长宽是否按照相同大小扩展
                  当长宽比相差比较大，直接scale会导致短边扩展不明显，
@@ -114,22 +115,30 @@ def extend_xyxy(xyxy: np.ndarray, scale=[1.0, 1.0], valid_range=[], fixed=False,
     """
     if len(xyxy) == 0 or len(scale) == 0: return xyxy
     if not isinstance(xyxy, np.ndarray): xyxy = np.asarray(xyxy)
-    cxcywh = xyxy.copy()
-    if fixed:
-        cxcywh[:, 0] = (xyxy[:, 2] + xyxy[:, 0]) / 2  # cx
-        cxcywh[:, 1] = (xyxy[:, 3] + xyxy[:, 1]) / 2  # cy
-        dw = (xyxy[:, 2] - xyxy[:, 0]) * (scale[0] - 1)
-        dh = (xyxy[:, 3] - xyxy[:, 1]) * (scale[1] - 1)
-        dp = np.vstack((dw, dh))
-        dp = np.max(dp, axis=0) if use_max else np.min(dp, axis=0)
-        cxcywh[:, 2] = (xyxy[:, 2] - xyxy[:, 0]) + dp  # w
-        cxcywh[:, 3] = (xyxy[:, 3] - xyxy[:, 1]) + dp  # h
-    else:
-        cxcywh[:, 0] = (xyxy[:, 2] + xyxy[:, 0]) / 2  # cx
-        cxcywh[:, 1] = (xyxy[:, 3] + xyxy[:, 1]) / 2  # cy
-        cxcywh[:, 2] = (xyxy[:, 2] - xyxy[:, 0]) * scale[0]  # w
-        cxcywh[:, 3] = (xyxy[:, 3] - xyxy[:, 1]) * scale[1]  # h
-    dxyxy = cxcywh2xyxy(cxcywh, width=None, height=None, normalized=False)
+    if len(scale) == 2:
+        cxcywh = xyxy.copy()
+        if fixed:
+            cxcywh[:, 0] = (xyxy[:, 2] + xyxy[:, 0]) / 2  # cx
+            cxcywh[:, 1] = (xyxy[:, 3] + xyxy[:, 1]) / 2  # cy
+            dw = (xyxy[:, 2] - xyxy[:, 0]) * (scale[0] - 1)
+            dh = (xyxy[:, 3] - xyxy[:, 1]) * (scale[1] - 1)
+            dp = np.vstack((dw, dh))
+            dp = np.max(dp, axis=0) if use_max else np.min(dp, axis=0)
+            cxcywh[:, 2] = (xyxy[:, 2] - xyxy[:, 0]) + dp  # w
+            cxcywh[:, 3] = (xyxy[:, 3] - xyxy[:, 1]) + dp  # h
+        else:
+            cxcywh[:, 0] = (xyxy[:, 2] + xyxy[:, 0]) / 2  # cx
+            cxcywh[:, 1] = (xyxy[:, 3] + xyxy[:, 1]) / 2  # cy
+            cxcywh[:, 2] = (xyxy[:, 2] - xyxy[:, 0]) * scale[0]  # w
+            cxcywh[:, 3] = (xyxy[:, 3] - xyxy[:, 1]) * scale[1]  # h
+        dxyxy = cxcywh2xyxy(cxcywh, width=None, height=None, normalized=False)
+    elif len(scale) == 4:
+        xywh = xyxy2xywh(xyxy)
+        xyxy[:, 0] = xyxy[:, 0] + xywh[:, 2] * (1 - scale[0])
+        xyxy[:, 1] = xyxy[:, 1] + xywh[:, 3] * (1 - scale[1])
+        xyxy[:, 2] = xyxy[:, 2] - xywh[:, 2] * (1 - scale[2])
+        xyxy[:, 3] = xyxy[:, 3] - xywh[:, 3] * (1 - scale[3])
+        dxyxy = xyxy
     if valid_range: dxyxy = clip_xyxy(dxyxy, valid_range=valid_range)
     return dxyxy
 
@@ -205,8 +214,8 @@ def get_square_boxes(boxes, use_max=True, use_mean=False, baseline=-1):
     """
     将boxes转换为正方形的boxes
     :param boxes:
-    :param use_max: 是否按照每个box(w,h)最大值(True)/最小值(False)进行转换(默认)
-    :param use_mean: 是否按照每个box(w,h)平均值进行转换(优先级比use_mean高)
+    :param use_max: 是否按照每个box(w,h)最大值(True)/最小值(False)进行转换(默认)，可以保证目标裁剪完整
+    :param use_mean: 是否按照每个box(w,h)平均值进行转换(优先级比use_max高),但当目标长宽比比较大时，可能出现目标裁剪不完整的问题
     :param baseline: 当baseline>0，表示正方形最小边长
     :return:
     """

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/cvutils/corner_utils.py

@@ -76,7 +76,7 @@ def get_target_points(src_pts: np.ndarray):
     return dst_pts
 
 
-def get_order_points(src_pts):
+def get_order_points_index(src_pts):
     """
     对4个点按顺时针方向进行排序:[top-left, top-right, bottom-right, bottom-left]
     top-left    ：对应y+x之和的最小点
@@ -91,13 +91,32 @@ def get_order_points(src_pts):
     :param src_pts: pts_dst [top-left, top-right, bottom-right, bottom-left]
     :return:
     """
-    dst_pts = np.zeros(shape=(4, 2), dtype=np.float32)
     s = src_pts.sum(axis=1)
-    dst_pts[0] = src_pts[np.argmin(s)]
-    dst_pts[2] = src_pts[np.argmax(s)]
     d = np.diff(src_pts, axis=1)
-    dst_pts[1] = src_pts[np.argmin(d)]
-    dst_pts[3] = src_pts[np.argmax(d)]
+    i0 = np.argmin(s)
+    i2 = np.argmax(s)
+    i1 = np.argmin(d)
+    i3 = np.argmax(d)
+    return [i0, i1, i2, i3]
+
+
+def get_order_points(src_pts):
+    """
+    对4个点按顺时针方向进行排序:[top-left, top-right, bottom-right, bottom-left]
+    top-left    ：对应y+x之和的最小点
+    bottom-right：对应y+x之和的最大点
+    top-right   ：对应y-x之差的最小点
+    bottom-left ：对应y-x之差的最大点
+         0(top-left)----(w10)----1(top-right)
+            |                       |
+          (h30)                    (h21)
+            |                       |
+        3(bottom-left)--(w23)---2(bottom-right)
+    :param src_pts: pts_dst [top-left, top-right, bottom-right, bottom-left]
+    :return:
+    """
+    index = get_order_points_index(src_pts)
+    dst_pts = src_pts[index]
     return dst_pts
 
 

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/cvutils/video_utils.py

@@ -61,7 +61,7 @@ def video2gif(video_file, gif_file=None, func=None, interval=1, use_pil=False, f
         image_utils.frames2gif_by_imageio(frames, gif_file=gif_file, fps=fps, loop=0)
 
 
-def video2frames(video_file, out_dir=None, func=None, interval=1, vis=True, delay=10):
+def video2frames(video_file, out_dir=None, func=None, interval=1, start=0, end=-1, vis=True, delay=10):
     """
     视频抽帧图像
     :param video_file: 视频文件
@@ -73,26 +73,14 @@ def video2frames(video_file, out_dir=None, func=None, interval=1, vis=True, dela
     """
     name = os.path.basename(video_file).split(".")[0]
     if not out_dir:  out_dir = os.path.join(os.path.dirname(video_file), name)
-    video_cap = get_video_capture(video_file)
-    width, height, num_frames, fps = get_video_info(video_cap)
-    if not interval: interval = fps
-    if not os.path.exists(out_dir): os.makedirs(out_dir)
-    count = 0
-    while True:
-        if count % interval == 0:
-            # 设置抽帧的位置
-            video_cap.set(cv2.CAP_PROP_POS_FRAMES, count)
-            isSuccess, frame = video_cap.read()
-            if not isSuccess or 0 < num_frames < count: break
-            if func:
-                frame = func(frame)
-            if vis:
-                image_utils.cv_show_image("frame", frame, use_rgb=False, delay=delay)
-            frame_file = os.path.join(out_dir, "{}_{:0=4d}.jpg".format(name, count))
-            cv2.imwrite(frame_file, frame)
-        count += 1
-    video_cap.release()
-    cv2.destroyAllWindows()
+    video_cap = video_iterator(video_file, save_video=None, interval=interval, start=start, end=end)
+    for data_info in video_cap:
+        frame = data_info["frame"]
+        count = data_info["count"]
+        if func: frame = func(frame)
+        if vis: image_utils.cv_show_image("frame", frame, use_rgb=False, delay=delay)
+        frame_file = os.path.join(out_dir, "{}_{:0=4d}.jpg".format(name, count))
+        cv2.imwrite(frame_file, frame)
 
 
 def video2frames_similarity(video_file, out_dir=None, func=None, interval=1, thresh=0.3, vis=True):
@@ -297,7 +285,7 @@ def video_iterator(video_file: int or str, save_video: str or int = None, interv
     w, h, num_frames, fps = image_utils.get_video_info(video_cap)
     start = int(kwargs.get("start", 0) * fps)
     end = int(kwargs.get("end", num_frames / fps) * fps) if fps > 0 else 0
-    end = min(end, num_frames)  # TODO 当num_frames<0时，使用0<end<count继续播放
+    end = min(end, num_frames) if end > 0 else num_frames  # TODO 当num_frames<0时，使用0<end<count继续播放
     interval = fps if interval == -1 else interval  # 当interval=-1，表示interval=fps,即一秒一帧
     save_fps = max(kwargs.get("speed", 1) * fps // interval, 1)
     count = 0

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/dataloader/base_dataset.py

@@ -78,7 +78,7 @@ class Dataset(object):
         else:
             class_dict = None
         if class_dict:
-            class_dict = json_utils.dict_sort_by_value(class_dict, reverse=False)
+            # class_dict = json_utils.dict_sort(class_dict, reverse=False)
             class_name = {}
             for n, i in class_dict.items():
                 class_name[i] = "{},{}".format(class_name[i], n) if i in class_name else n
@@ -129,23 +129,16 @@ class Dataset(object):
         return Dataset.read_file(filename, split=split)
 
     @staticmethod
-    def get_file_list(dir_root, postfix=['*.jpg'], basename=True):
+    def get_file_list(dir_root, postfix=['*.jpg'], sub=False, basename=True):
         """
         获得文件列表
         :param dir_root: 图片文件目录
         :param postfix: 后缀名，可是多个如，['*.jpg','*.png']
+        :param sub: 是否去除根路径
         :param basename: 返回的列表是文件名（True），还是文件的完整路径(False)
         :return:
         """
-        file_list = []
-        for format in postfix:
-            format = os.path.join(dir_root, format)
-            image_list = glob.glob(format)
-            if image_list:
-                file_list += image_list
-        file_list = sorted(file_list)
-        if basename:
-            file_list = [os.path.basename(f).split(".")[0] for f in file_list]
+        file_list = file_utils.get_files_list(dir_root, prefix="", postfix=postfix, sub=sub, basename=basename)
         return file_list
 
     @staticmethod

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/dataloader/data_resample.py

@@ -178,6 +178,8 @@ class DataResample(object):
                 class_count[label] += 1
             except Exception as e:
                 class_count[label] = 1
+        from pybaseutils import json_utils
+        class_count = json_utils.dict_sort(class_count,use_key=True)
         return class_count
 
     @staticmethod

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/dataloader/parser_image_folder.py

@@ -50,7 +50,7 @@ class FolderDataset(parser_image_text.TextDataset):
         保存格式：[path,label] 或者 [path,label,xmin,ymin,xmax,,ymax]
         :param data_file:
         :param data_root:
-        :return: item_list [{"file":file,"label":label}]
+        :return: item_list [{"file","label","name","bbox"}],bbox非必须
         """
         if isinstance(data_file, str): data_file = [data_file]
         item_list = []
@@ -61,7 +61,7 @@ class FolderDataset(parser_image_text.TextDataset):
             print("loading data from:{},have {},label:{}".format(dir, len(paths), len(set(labels))))
             # TODO # 避免多个数据集的相同的label
             if use_sub:  labels = [os.path.join(str(i), l) for l in labels]
-            data = [{"file": p, "label": l} for p, l in zip(paths, labels)]
+            data = [{"file": p, "label": l, 'name': l} for p, l in zip(paths, labels)]
             item_list += data
         return item_list
 
@@ -70,7 +70,7 @@ if __name__ == '__main__':
     from pybaseutils import image_utils
     from torchvision import transforms
 
-    image_dir = ['/home/PKing/nasdata/release/infrastructure/DMClassification/data/dataset/train']
+    image_dir = ['/home/PKing/nasdata/release/infrastructure/DMClassification/data/dataset/test']
     class_name = []
     input_size = [224, 224]
     rgb_mean = [0., 0., 0.]
@@ -84,7 +84,7 @@ if __name__ == '__main__':
                             transform=transform,
                             shuffle=True,
                             class_name=class_name,
-                            resample=True,
+                            resample=False,
                             disp=True)
     for i in range(len(dataset)):
         data_info = dataset.__getitem__(i)

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/dataloader/parser_image_text.py

@@ -13,7 +13,7 @@ sys.path.append(os.getcwd())
 import PIL.Image as Image
 import numpy as np
 import random
-import math
+import time
 import cv2
 from tqdm import tqdm
 from pybaseutils.dataloader.base_dataset import Dataset, ConcatDataset
@@ -36,17 +36,26 @@ class TextDataset(Dataset):
         :param phase:
         :param disp:
         :param check:
-        :param kwargs: use_max,use_mean,crop_scale,resample,save_info
+        :param kwargs:  log: print or log.info
+                        use_max,
+                        use_mean,
+                        crop_scale,
+                        resample,
+                        save_info,
+                        resample_interval
         """
         self.data_file = data_file
         self.data_root = data_root
         self.use_rgb = use_rgb
         self.transform = transform
         self.phase = phase
+        self.train = self.phase.lower() == "train"
         self.shuffle = shuffle
         self.check = check
         self.kwargs = kwargs
+        self.log = kwargs.get('log', print) if kwargs.get('log', print) else print
         self.label_index = kwargs.get("label_index", "label")  # 类别字段key
+        self.crop_scale = kwargs.pop("crop_scale", [])  # TODO bbox缩放系数
         self.class_name, self.class_dict = self.parser_classes(class_name)
         self.item_list = self.parser_dataset(data_file, data_root=data_root, label_index=self.label_index,
                                              shuffle=shuffle, check=check)
@@ -64,16 +73,19 @@ class TextDataset(Dataset):
         self.classes = list(self.class_dict.values())
         self.num_classes = max(self.classes) + 1
         self.num_samples = len(self.item_list)
-        self.info(save_info=kwargs.get("save_info", ""))
+        self.t0 = time.time()
+        self.resample_interval = kwargs.get('resample_interval', 60)  # TODO 重采样间隔，低于该时间的不进行重采集，避免频繁采样
+        if self.log: self.info(save_info=kwargs.get("save_info", ""))
 
     def info(self, save_info=""):
-        print("----------------------- {} DATASET INFO -----------------------".format(self.phase.upper()))
-        print("Dataset num_samples   :{}".format(len(self.item_list)))
-        print("Dataset num_classes   :{}".format(self.num_classes))
-        print("Dataset class_name    :{}".format(self.class_name))
-        print("Dataset class_dict    :{}".format(self.class_dict))
-        print("Dataset class_count   :{}".format(self.class_count))
-        print("Dataset resample      :{}".format(self.resample))
+        self.log("----------------------- {} DATASET INFO -----------------------".format(self.phase.upper()))
+        self.log("Dataset kwargs        :{}".format(self.kwargs))
+        self.log("Dataset num_samples   :{}".format(len(self.item_list)))
+        self.log("Dataset num_classes   :{}".format(self.num_classes))
+        self.log("Dataset class_name    :{}".format(self.class_name))
+        self.log("Dataset class_dict    :{}".format(self.class_dict))
+        self.log("Dataset class_count   :{}".format(self.class_count))
+        self.log("Dataset resample      :{}".format(self.resample))
         if save_info:
             if not os.path.exists(save_info): os.makedirs(save_info)
             m = np.mean(list(self.class_count.values()))
@@ -85,8 +97,8 @@ class TextDataset(Dataset):
             file_utils.save_json(os.path.join(save_info, f"{self.phase}_class_count.json"), self.class_count)
             file_utils.write_list_data(os.path.join(save_info, f"{self.phase}_class_name.txt"), self.class_name)
             file_utils.save_json(os.path.join(save_info, f"{self.phase}_class_lack.json"), class_lack)
-            print("loss_labels: {}".format(class_lack))
-        print("------------------------------------------------------------------")
+            self.log("loss_labels: {}".format(class_lack))
+        self.log("------------------------------------------------------------------")
 
     def parser_dataset(self, data_file, data_root="", label_index="label", shuffle=False, check=False):
         """
@@ -101,6 +113,7 @@ class TextDataset(Dataset):
         data_list = self.load_dataset(data_file, data_root=data_root)
         if not self.class_name:
             self.class_name = list(set([d[label_index] for d in data_list]))
+            self.class_name = sorted(self.class_name)
             self.class_name, self.class_dict = self.parser_classes(self.class_name)
         item_list = []
         for data in data_list:
@@ -118,10 +131,10 @@ class TextDataset(Dataset):
 
     def load_dataset(self, data_file, data_root="", **kwargs):
         """
-        保存格式：[path,label] 或者 [path,label,xmin,ymin,xmax,ymax]
+        txt保存格式：[path,name] 或者 [path,name,xmin,ymin,xmax,ymax]
         :param data_file:
         :param data_root:
-        :return: item_list [{"file":file,"label":label},"bbox":[]]
+        :return: item_list [{"file","label","name","bbox"}],bbox非必须
         """
         if isinstance(data_file, str): data_file = [data_file]
         item_list = []
@@ -130,10 +143,9 @@ class TextDataset(Dataset):
             content = file_utils.read_data(file, split=",")
             data = []
             for line in content:
-                if len(line) == 2:
-                    data.append({"file": os.path.join(root, line[0]), "label": line[1]})
-                elif len(line) == 6:
-                    data.append({"file": os.path.join(root, line[0]), "label": line[1], "bbox": line[2:]})
+                item = {"file": os.path.join(root, line[0]), "label": line[1], 'name': line[1]}
+                if len(line) == 6: item['bbox'] = line[2:]  # (xmin,ymin,xmax,ymax)
+                data.append(item)
             print("loading data from:{},have {}".format(file, len(data)))
             item_list += data
         return item_list
@@ -157,33 +169,41 @@ class TextDataset(Dataset):
     def __getitem__(self, index):
         """
         :param index:
-        :return: {"image": image, "label": label}
+        :return: {"image","label","name"}
         """
         item = self.item_list[index]
-        file, label, bbox = item["file"], item[self.label_index], item.get("bbox", [])
-        name = self.class_name[label]
+        file, label, name, bbox = item["file"], item[self.label_index], item['name'], item.get("bbox", [])
         image = self.read_image(file, use_rgb=self.use_rgb)
-        image = self.crop_image(image, bbox=bbox, **self.kwargs) if bbox else image
+        image = self.crop_image(image, bbox=bbox, crop_scale=self.crop_scale, **self.kwargs)
         if self.transform:
-            image = Image.fromarray(image)
-            image = self.transform(image)
+            image = self.transform(Image.fromarray(image))
         if image is None:
             index = int(random.uniform(0, self.num_samples))
             return self.__getitem__(index)
         return dict(image=image, label=label, file=file, name=name)
 
     def __len__(self):
-        if self.resample:
+        self.t1 = time.time()  # seconds
+        dt = (self.t1 - self.t0)
+        if dt > self.resample_interval and self.resample:  # 如果时间间隔太小则不进行重采样
+            print(f"resample {self.phase} dataset")
             self.item_list = self.data_resample.update(True)
+            self.t0 = self.t1
         return len(self.item_list)
 
-    def crop_image(self, image, bbox, **kwargs):
-        """裁剪图片"""
+    def crop_image(self, image, bbox, crop_scale=[], use_max=False, use_mean=True, **kwargs):
+        """
+        裁剪图片
+        :param image:
+        :param bbox:
+        :param kwargs: use_max,use_mean,crop_scale
+                use_max: 是否按照每个box(w,h)最大值(True)/最小值(False)进行转换(默认)，可以保证目标裁剪完整
+                use_mean: 是否按照每个box(w,h)平均值进行转换(优先级比use_max高),但当目标长宽比比较大时，可能出现目标裁剪不完整的问题
+        :return:
+        """
         if len(bbox) == 0: return image
-        boxes = image_utils.get_square_boxes(boxes=[bbox],
-                                             use_max=kwargs.get("use_max", False),
-                                             use_mean=kwargs.get("use_mean", True))
-        boxes = image_utils.extend_xyxy(boxes, scale=kwargs.get("crop_scale", []))
+        boxes = image_utils.get_square_boxes(boxes=[bbox], use_max=use_max, use_mean=use_mean)
+        boxes = image_utils.extend_xyxy(boxes, scale=crop_scale)
         image = image_utils.get_boxes_crop(image, boxes)[0]
         return image
 
@@ -236,6 +256,7 @@ class TextDataset(Dataset):
         for item in item_list:
             label = item[label_index]
             count[label] = count[label] + 1 if label in count else 1
+        count = json_utils.dict_sort(count, use_key=True)
         if class_name: count = {class_name[k]: v for k, v in count.items()}
         return count
 
@@ -245,7 +266,7 @@ if __name__ == '__main__':
     from torchvision import transforms
 
     data_files = [
-        '/home/PKing/nasdata/tmp/tmp/RealFakeFace/anti-spoofing-images-v2/train.txt',
+        '/home/PKing/nasdata/tmp/tmp/RealFakeFace/anti-spoofing/anti-spoofing-images-v2/test.txt',
     ]
     class_name = None
     input_size = [112, 112]

{pybaseutils-2.2.22 → pybaseutils-2.2.25}/pybaseutils/dataloader/parser_labelme.py

@@ -94,8 +94,7 @@ class LabelMeDataset(Dataset):
         :return:
         """
         image_file = os.path.join(image_dir, image_name)
-        img_postfix = image_name.split(".")[-1]
-        image_id = image_name[:-len(img_postfix) - 1]
+        image_id, img_postfix = file_utils.split_postfix(image_file)
         anno_file = os.path.join(anno_dir, "{}.json".format(image_id))
         return image_file, anno_file, image_name
 
@@ -150,11 +149,11 @@ class LabelMeDataset(Dataset):
         if not image_dir:
             image_dir = self.search_path(data_root, ["images", "JPEGImages"])
         if image_dir and not image_ids:
-            image_ids = self.get_file_list(image_dir, postfix=file_utils.IMG_POSTFIX, basename=False)
-            image_ids = [os.path.basename(f) for f in image_ids]
+            image_ids = self.get_file_list(image_dir, postfix=file_utils.IMG_POSTFIX, sub=True, basename=False)
+            if not anno_dir: anno_dir = image_dir
         elif anno_dir and not image_ids:
-            image_ids = self.get_file_list(anno_dir, postfix=["*.json", "*.xml"], basename=False)
-            image_ids = [os.path.basename(f) for f in image_ids]
+            image_ids = self.get_file_list(anno_dir, postfix=file_utils.IMG_POSTFIX, sub=True, basename=False)
+            if not image_dir: image_dir = anno_dir
         assert isinstance(anno_dir, str) and os.path.exists(anno_dir), "no anno_dir :{}".format(anno_dir)
         assert isinstance(image_dir, str) and os.path.exists(image_dir), "no image_dir:{}".format(image_dir)
         assert len(image_ids) > 0, f"image_ids is empty,image_dir={image_dir},anno_dir={anno_dir}"
@@ -330,7 +329,7 @@ class LabelMeDataset(Dataset):
             width = annotation.get('imageWidth', -1)
             height = annotation.get('imageHeight', -1)
         except:
-            print("illegal annotation:{}".format(anno_file))
+            # print("illegal annotation:{}".format(anno_file))
             annos = []
             width = -1
             height = -1
@@ -487,8 +486,7 @@ def show_target_image(image, boxes, labels, points, keypoints=[], color=(), thic
 if __name__ == "__main__":
     from pybaseutils.converter import build_labelme
 
-    anno_dir = "/home/PKing/Downloads/sample/images"
-    # anno_dir = "/home/PKing/Downloads/冲击试验/sample/images"
+    anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-action-cvlm-v2/train-v2/01-核相操作/dataset-v01/images"
     names = None
     dataset = LabelMeDatasets(filename=None,
                               data_root=None,