pybaseutils 2.0.6__tar.gz → 2.0.8__tar.gz

{pybaseutils-2.0.6 → pybaseutils-2.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybaseutils
-Version: 2.0.6
+Version: 2.0.8
 Summary: pybaseutils
 Home-page: https://github.com/PanJinquan/base-utils
 Author: PanJinquan

{pybaseutils-2.0.6 → pybaseutils-2.0.8}/pybaseutils/__init__.py

@@ -5,4 +5,4 @@
     @Date   : 2019-05-07 17:40:27
 """
 
-__version__ = '2.0.6'
+__version__ = '2.0.8'

pybaseutils-2.0.8/pybaseutils/dataloader/balanced_classes.py

@@ -0,0 +1,222 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : panjq
+    @E-mail : pan_jinquan@163.com
+    @Date   : 2019-11-18 14:08:46
+"""
+import numpy as np
+import math
+
+
+def get_torch_sample(weights,
+                     num_samples: int,
+                     replacement: bool = True,
+                     generator=None):
+    '''
+    https://blog.csdn.net/caihuanqia/article/details/113258690
+    :param weights:weights参数对应的是“样本”的权重而不是“类别的权重”,权重越大，采样次数更多
+    :param num_samples:
+    :param replacement:
+    :param generator:
+    :return:
+    '''
+    import torch.utils.data as torch_utils
+    sampler = torch_utils.sampler.WeightedRandomSampler(weights, len(weights))
+    return sampler
+
+
+def class_weight_to_sample_weight(labels_list: list, class_weight: dict):
+    '''
+    :param labels_list:lable必须从0开始的,连续的int类型
+    :param class_weight:
+    :return:
+    '''
+    sample_weight = [0] * len(labels_list)
+    for idx, name in enumerate(labels_list):
+        sample_weight[idx] = class_weight[name]
+    return sample_weight
+
+
+def count_class_samples_nums(labels_list):
+    '''
+    classes_dict = {cls_id0: nums_of_id0,cls_id1: nums_of_id1,...,}
+    classes_dict = {0: 5,1: 5, 2: 2, 3: 2, 4: 4}
+    =========
+    # nclasses = len(set(labels_list)) # fix a BUG
+    nclasses = max(labels_list) + 1
+    count = [0] * nclasses
+    for name in labels_list:
+        count[name] += 1  # item is (img-data, label-id)
+    classes_dict = dict(enumerate(count))
+    =========
+    :param labels_list:lable必须从0开始的,连续的int类型
+    :return:
+    '''
+    count_class = {}
+    for name in labels_list:
+        try:
+            count_class[name] += 1
+        except Exception as e:
+            count_class[name] = 1
+    return count_class
+
+
+def create_class_sample_weight_sklearn(labels_list: list, balanced='balanced', weight_type="class_weight"):
+    '''
+    balanced : dict, 'balanced' or None
+    If 'balanced', class weights will be given by
+    ``n_samples / (n_classes * np.bincount(lt_steps))``.
+    If a dictionary is given, keys are classes and values are corresponding class weights.
+    If None
+    :param labels_list: lable必须从0开始的,连续的int类型
+    :param balanced:dict, 'balanced' or None
+            If 'balanced', class weights will be given by
+            ``n_samples / (n_classes * np.bincount(lt_steps))``.
+            If a dictionary is given, keys are classes and values
+            are corresponding class weights.
+            If None is given, the class weights will be uniform.
+    :param weight_type: class_weight or sample_weight
+    :return:
+    '''
+    import sklearn
+    classes = np.unique(labels_list)
+    weight_list = sklearn.utils.class_weight.compute_class_weight(balanced,
+                                                                  classes,
+                                                                  labels_list)
+    if weight_type == "class_weight":
+        class_weight = dict(zip([x for x in classes], weight_list))
+        return class_weight
+    elif weight_type == "sample_weight":
+        class_weight = dict(zip([x for x in classes], weight_list))
+        sample_weight = class_weight_to_sample_weight(labels_list, class_weight)
+        return sample_weight
+    else:
+        return weight_list
+
+
+def create_class_sample_weight_custom(labels_list, balanced="balanced", weight_type="class_weight"):
+    '''
+
+    :param labels_list:lable必须从0开始的,连续的int类型
+    :param balanced:
+    :param weight_type: class_weight:返回每个classs的权重
+                        sample_weight:返回每个labels_list对应的权重
+    :param mu:
+    :return:
+    '''
+    count_class = count_class_samples_nums(labels_list)
+    n_samples = np.sum(list(count_class.values()))
+    classes = count_class.keys()
+    n_classes = len(classes)
+    class_weight = dict()
+    weight = sum(class_weight.values())
+    # 计算每个类别的权重：样本越少，权重越大
+    for cls in classes:
+        cls_num = float(count_class[cls])
+        if balanced == "log_balanced":
+            mu = 0.15
+            score = math.log(mu * n_samples / cls_num)
+            class_weight[cls] = score if score > 1.0 else 1.0
+        elif balanced == "balanced":
+            # score = n_samples / (n_classes * np.bincount(lt_steps))
+            score = n_samples / (n_classes * cls_num)
+            class_weight[cls] = score
+        elif balanced == "auto":
+            # N / float(count[i])
+            score = n_samples / cls_num
+            class_weight[cls] = score
+        else:
+            raise Exception("Error:{}".format(balanced))
+    # loss_weight = {k: v / weight for k, v in class_weight.items()}
+    if weight_type == "class_weight":
+        return class_weight
+    elif weight_type == "sample_weight":
+        sample_weight = class_weight_to_sample_weight(labels_list, class_weight)
+        return sample_weight
+    else:
+        raise Exception("Error:{}".format(weight_type))
+
+
+def create_sample_weight_torch(labels_list, nclasses=None):
+    '''
+    Make a vector of weights for each image in the dataset, based
+    on class frequency. The returned vector of weights can be used
+    to create a WeightedRandomSampler for a DataLoader to have
+    class balancing when sampling for a training batch.
+        images - torchvisionDataset.imgs
+        nclasses - len(torchvisionDataset.classes)
+    https://discuss.pytorch.org/t/balanced-sampling-between-classes-with-torchvision-dataloader/2703/3
+    labels_list = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4]
+    weight       =[3.6, 3.6, 3.6, 3.6, 3.6, 3.6, 3.6, 3.6, 3.6, 3.6, 9.0, 9.0, 9.0, 9.0, 4.5, 4.5, 4.5, 4.5]
+    '''
+    if not nclasses:
+        nclasses = len(set(labels_list))
+    classes_dict = count_class_samples_nums(labels_list)
+    count = list(classes_dict.values())
+    weight_per_class = [0.] * nclasses
+    N = float(sum(count))  # total number of images
+    for i in range(nclasses):
+        weight_per_class[i] = N / float(count[i])
+    weight = [0] * len(labels_list)
+    for idx, name in enumerate(labels_list):
+        weight[idx] = weight_per_class[name]
+
+    return weight
+
+
+def keras_example(class_weight, sample_weight):
+    '''
+    在keras,fit函数中调用class_weight，可以通过字典设置每个类别输入权重，比如：cw = {0: 1, 1: 25}，
+    类别序列可以使用.class_indices获取。
+    :param model:
+    :param class_weight: Optional dictionary mapping class indices (integers)
+        to a weight (float) value, used for weighting the loss function
+        (during training only).
+        This can be useful to tell the model to
+        "pay more attention" to samples from
+        an under-represented class.
+    :param sample_weight: type->numpy array，用于在训练时调整损失函数（仅用于训练）。
+    可以传递一个1D的与样本等长的向量用于对样本进行1对1的加权，或者在面对时序数据时，
+    传递一个的形式为（samples，sequence_length）的矩阵来为每个时间步上的样本赋不同的权。
+    这种情况下请确定在compile时添加了sample_weight_mode=‘temporal’。
+        1.sample_weight会覆盖class_weight，所以二者用其一；
+        2.如果仅仅是类不平衡，则使用class_weight，sample_weights则是类内样本之间还不平衡的时候使用。
+        3.keras已经在新版本中加入了 class_weight = ‘auto’。
+        设置了这个参数后，keras会自动设置class weight让每类的sample对损失的贡献相等。
+        4.在设置类别权重的时候，类别序列可以使用train_generator.class_indices获取。
+    :return:
+    '''
+    import tensorflow as tf
+    train_dataset = ...
+    steps_per_epoch = ...
+    model = tf.keras.models.Sequential([
+        tf.keras.layers.Dense(512, activation=None, name="fc1")
+    ])
+    model.compile(optimizer='adam',
+                  loss='sparse_categorical_crossentropy',
+                  metrics=['accuracy'])
+    model.fit(train_dataset,
+              steps_per_epoch=steps_per_epoch,
+              sample_weight=sample_weight,
+              class_weight='auto')
+
+
+if __name__ == "__main__":
+    # random labels_dict
+    # labels_dict = {0: 2813, 1: 78, 2: 2814, 3: 78, 4: 7914, 5: 248, 6: 7914, 7: 248}
+    labels_list = [4, 1, 2, 3, 2, 3, 3, 4, 4, 4, 0]
+    print("create_class_sample_weight_custom-------------------------")
+    w1 = create_class_sample_weight_custom(labels_list, balanced="balanced", weight_type="class_weight")
+    w2 = create_class_sample_weight_custom(labels_list, balanced="balanced", weight_type="sample_weight")
+    print("class_weight :{}".format(w1))
+    print("labels_list  :{}".format(labels_list))
+    print("sample_weight:{}".format(w2))
+    print("create_class_sample_weight_sklearn-------------------------")
+    w2 = create_class_sample_weight_sklearn(labels_list, balanced="balanced", weight_type="class_weight")
+    w3 = create_class_sample_weight_sklearn(labels_list, balanced="balanced", weight_type="sample_weight")
+    print("class_weight :{}".format(w2))
+    print("labels_list  :{}".format(labels_list))
+    print("sample_weight:{}".format(w3))
+    # print(class_weight_to_sample_weight(labels_list, w2))
+    # w3 = create_sample_weight_torch(labels_list, nclasses=5)
+    # print(w3)

{pybaseutils-2.0.6 → pybaseutils-2.0.8}/pybaseutils/dataloader/base_coco.py

@@ -200,7 +200,7 @@ class CocoDataset(object):
         :return:
         """
         if isinstance(class_name, str):
-            class_name = Dataset.read_files(class_name)
+            class_name = Dataset.read_file(class_name)
         elif isinstance(class_name, list) and "unique" in class_name:
             self.unique = True
         if isinstance(class_name, list) and len(class_name) > 0:

{pybaseutils-2.0.6 → pybaseutils-2.0.8}/pybaseutils/dataloader/base_dataset.py

@@ -66,7 +66,7 @@ class Dataset(object):
         :return:
         """
         if isinstance(class_name, str):
-            class_name = Dataset.read_files(class_name)
+            class_name = Dataset.read_file(class_name)
         elif isinstance(class_name, list) and "unique" in class_name:
             self.unique = True
         if isinstance(class_name, list) and len(class_name) > 0:
@@ -109,7 +109,7 @@ class Dataset(object):
         return json_data
 
     @staticmethod
-    def read_files(filename, split=None):
+    def read_file(filename, split=None):
         """
         :param filename:
         :param split:分割

pybaseutils-2.0.8/pybaseutils/dataloader/data_resample.py

@@ -0,0 +1,220 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : Pan
+    @E-mail :
+    @Date   : 2021-04-21 09:41:34
+"""
+import random
+import numpy as np
+import math
+
+
+class ResampleExample(object):
+    """样本均衡，重采样DataResampler的使用方法"""
+
+    def __init__(self, item_list, label_index=1, shuffle=True, disp=False):
+        """
+        :param item_list: item_list=[item_0,item_1,...,item_n],
+                          item_n= [path/to/image,label]
+        :param label_index: label在item_n的index
+        :param disp: 是否打印log信息
+        """
+        self.disp = disp
+        self.shuffle = shuffle
+        self.item_list = item_list
+        self.resampler = DataResample(self.item_list,
+                                      label_index=label_index,
+                                      shuffle=self.shuffle,
+                                      disp=self.disp)
+
+    def __len__(self):
+        # 更新resampler，实现每个epoch重新采样，避免样本数比较多的类别，没有加入训练
+        self.item_list = self.resampler.update(self.shuffle)
+        return len(self.item_list)
+
+    def __getitem__(self, idx):
+        image_path = self.item_list[idx][0]
+        label_id = self.item_list[idx][1]
+        return image_path, label_id
+
+
+class DataResample(object):
+    """样本均衡，重采样的方法"""
+
+    def __init__(self, item_list=[], label_index=1, balance="mean", shuffle=True, disp=False):
+        """
+        Usage:
+        参考：ResampleExample例子的使用方法
+        :param item_list:
+        :param label_index:
+        :param balance:实现样本均衡策略,均衡力度：mean > log > sqrt > y
+                        "y": 每个label样本数跟原来一样
+                        "sqrt": 每个label样本取sqrt数，实现样本均衡
+                        "log": 每个label样本取log数，实现样本均衡
+                        "mean": 每个label样本取样本平均数，每个label的个数一样
+        """
+        self.src_item_list = item_list
+        self.label_index = label_index
+        self.balance = balance
+        self.shuffle = shuffle
+        self.disp = disp
+        self.class_count = self.get_class_count(self.src_item_list, label_index)
+        self.class_item_dict = self.get_class_item_dict(self.src_item_list, label_index)
+        self.balance_nums = self.get_balance_nums(self.class_count, self.balance)
+        self.item_list = self.update(shuffle=self.shuffle)
+        self.class_weight = self.get_class_weight(self.class_count)
+
+    def __len__(self):
+        self.update(shuffle=self.shuffle)
+        return len(self.item_list)
+
+    def update(self, shuffle=False):
+        self.item_list = self.get_resample_data(shuffle=shuffle)
+        return self.item_list
+
+    def get_resample_data(self, shuffle=True):
+        """
+        获得重采样的数据
+        :param item_list:
+        :param label_index:
+        :param shuffle:
+        :return:
+        """
+        if self.disp:
+            print("class_item_dict:{}".format({k: len(v) for k, v in self.class_item_dict.items()}))
+        out_list = []
+        for name, per_class_list in self.class_item_dict.items():
+            nums = self.balance_nums[name]
+            per_list = self.get_sampler(per_class_list, nums, shuffle=shuffle)
+            out_list += per_list
+        if shuffle:
+            random.shuffle(out_list)
+        if self.disp:
+            # 统计每个类别的个数
+            class_count = self.get_class_count(out_list, self.label_index)
+            print("resampler count_class :{},total:{}".format(class_count, sum(class_count.values())))
+        return out_list
+
+    def get_balance_nums(self, class_count: dict, balance):
+        """
+        获得平衡后，每个样本的数目
+        :param class_count:
+        :param balance:
+        :return:
+        """
+        class_name = list(class_count.keys())
+        num_samples = sum(class_count.values())  # 总样本数目
+        if balance == "mean":
+            mean_samples = num_samples * 1.0 / len(class_name)  # 平均样本数
+            balance_nums = {name: mean_samples for name, c in class_count.items()}
+        elif balance == "log":
+            # Fix Bug:c=1
+            balance_nums = {name: np.log(c + 1) for name, c in class_count.items()}
+        elif balance == "sqrt":
+            # Fix Bug:c=0
+            balance_nums = {name: np.sqrt(c + 1) for name, c in class_count.items()}
+        elif balance == "y":
+            balance_nums = {name: c for name, c in class_count.items()}
+        else:
+            raise Exception("Error:{}".format(balance))
+        sum_balance = sum(balance_nums.values())
+        balance_nums = {name: math.ceil(c / sum_balance * num_samples) for name, c in balance_nums.items()}
+        return balance_nums
+
+    def get_sampler(self, item_list, nums, shuffle=True):
+        """
+        提取nums个数，不足nums个时，会进行填充
+        :param item_list: 输入样本列表
+        :param nums: 需要提取的样本数目
+        :param shuffle: 是否随机提取样本
+        :return:
+        """
+        item_nums = len(item_list)
+        if nums > item_nums:
+            item_list = item_list * math.ceil(nums / item_nums)
+        if shuffle:
+            random.shuffle(item_list)
+        out_list = item_list[:nums]
+        return out_list
+
+    @staticmethod
+    def get_label_list(item_list, label_index):
+        labels_list = []
+        for item in item_list:
+            label = item[label_index]
+            labels_list.append(label)
+        return labels_list
+
+    @staticmethod
+    def get_class_item_dict(item_list, label_index):
+        """
+        获得每一类的样本
+        :param item_list:
+        :return:
+        """
+        class_item_dict = {}
+        for item in item_list:
+            label = item[label_index]
+            try:
+                # if label in class_item_dict: # 比较慢，相当于需要查询label是否存在
+                class_item_dict[label] += [item]
+            except Exception as e:
+                class_item_dict[label] = [item]
+        return class_item_dict
+
+    @staticmethod
+    def get_class_count(item_list, label_index):
+        """
+        统计每个类别的个数
+        :param item_list:
+        :param label_index: label在item中的序号
+        :return:
+        """
+        class_count = {}
+        for item in item_list:
+            label = item[label_index]
+            try:
+                # if label in class_count:  # 比较慢，相当于需要查询label是否存在
+                class_count[label] += 1
+            except Exception as e:
+                class_count[label] = 1
+        return class_count
+
+    @staticmethod
+    def get_class_weight(class_count: dict):
+        """
+        计算每个label的权重，类别越少，权重越大
+        :param class_count:
+        :return:
+        """
+        n_samples = sum(list(class_count.values()))
+        class_weight = {}
+        for cls, num in class_count.items():
+            score = n_samples / num
+            class_weight[cls] = score
+        return class_weight
+
+
+if __name__ == "__main__":
+    from torch.utils.data import Dataset, DataLoader, Sampler
+    from utils import torch_tools
+
+    torch_tools.set_env_random_seed()
+    label0 = [["0.1.jpg", 0], ["0.2.jpg", 0], ["0.3.jpg", 0]]
+    label1 = [["1.jpg", 1]] * 5
+    label2 = [["2.0.jpg", 2], ["2.1.jpg", 2], ["2.2.jpg", 2], ["2.3.jpg", 2], ["2.4.jpg", 2],
+              ["2.5.jpg", 2], ["2.6.jpg", 2], ["2.7.jpg", 2], ["2.8.jpg", 2]]
+    label3 = [["3.1.jpg", 3], ["3.2.jpg", 3], ["3.3.jpg", 3]]
+    item_list = label0 + label1 + label2 + label3
+    item_list = item_list * 1000000
+    print("have item_list:{}".format(len(item_list)))
+    dataset_train = ResampleExample(item_list=item_list, label_index=1, shuffle=True, disp=True)
+    # dataset_train = ResampleExample(item_list=item_list, label_index=1, disp=False)
+    # dataset_train = ResampleExample(item_list=item_list, label_index=1, disp=False)
+    batch_size = 4
+    dataloader = DataLoader(dataset_train, batch_size, num_workers=0)
+    epochs = 2
+    for epoch in range(epochs):
+        print("{}===".format(epoch) * 10)
+        for batch_image, batch_label in iter(dataloader):
+            print(batch_image, batch_label)

pybaseutils-2.0.8/pybaseutils/dataloader/parser_image_folder.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+# --------------------------------------------------------
+# @Author : panjq
+# @Date   : 2019-9-20 13:18:34
+# --------------------------------------------------------
+"""
+
+import os
+import cv2
+import PIL.Image as Image
+import numpy as np
+import random
+from pybaseutils import image_utils, file_utils
+from pybaseutils.dataloader import parser_image_text
+
+
+class FolderDataset(parser_image_text.TextDataset):
+    def __init__(self, image_dir, class_name=None, transform=None, use_rgb=False, shuffle=False,
+                 phase="test", disp=False, check=False, **kwargs):
+        """
+        :param image_dir: [image_dir]->list or `path/to/image_dir`->str
+        :param class_name:
+        :param transform: torch transform
+        :param shuffle:
+        :param disp:
+        """
+        super(FolderDataset, self).__init__(filename=image_dir,
+                                            data_root=None,
+                                            class_name=class_name,
+                                            transform=transform,
+                                            shuffle=shuffle,
+                                            use_rgb=use_rgb,
+                                            phase=phase,
+                                            disp=disp,
+                                            check=check,
+                                            **kwargs)
+
+    def __getitem__(self, index):
+        """
+        :param index:
+        :return: image,label
+        """
+        item = self.item_list[index]
+        bbox = item[2:] if len(item) == 6 else []
+        image_file, label = item[0], item[1]
+        image = self.read_image(image_file, use_rgb=self.use_rgb)
+        image = self.crop_image(image, bbox=bbox) if bbox else image
+        if self.transform:
+            image = Image.fromarray(image)
+            image = self.transform(image)
+        if image is None:
+            index = int(random.uniform(0, self.num_images))
+            return self.__getitem__(index)
+        return {"image": image, "label": label}
+
+    def load_dataset(self, filename, data_root="", use_sub=False):
+        """
+        保存格式：[path,label] 或者 [path,label,xmin,ymin,xmax,,ymax]
+        :param filename:
+        :param data_root:
+        :return: item_list [path,label] 或者 [path,label,xmin,ymin,xmax,,ymax]
+        """
+        if isinstance(filename, str): filename = [filename]
+        item_list = []
+        for i, dir in enumerate(filename):
+            print("loading image from:{}".format(dir))
+            if not os.path.exists(dir): raise Exception("image_dir:{}".format(dir))
+            paths, labels = file_utils.get_files_labels(dir, postfix=file_utils.IMG_POSTFIX)
+            # TODO # 避免多个数据集的相同的label
+            if use_sub:  labels = [os.path.join(str(i), l) for l in labels]
+            data = [[p, l] for p, l in zip(paths, labels)]
+            item_list += data
+        return item_list
+
+
+if __name__ == '__main__':
+    from pybaseutils import image_utils
+    from torchvision import transforms
+
+    image_dir = ['/home/PKing/nasdata/release/infrastructure/DMClassification/data/dataset/train']
+    input_size = [224, 224]
+    rgb_mean = [0., 0., 0.]
+    rgb_std = [1.0, 1.0, 1.0]
+    transform = transforms.Compose([
+        transforms.Resize(input_size),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=rgb_mean, std=rgb_std),
+    ])
+    dataset = FolderDataset(image_dir=image_dir,
+                            transform=transform,
+                            shuffle=True,
+                            class_name=None,
+                            resample=True,
+                            disp=True)
+    for i in range(len(dataset)):
+        data_info = dataset.__getitem__(i)
+        image, label = data_info["image"], data_info["label"]
+        image = np.asarray(image).transpose(1, 2, 0)  # 通道由[c,h,w]->[h,w,c]
+        image = np.asarray(image * 255, dtype=np.uint8)
+        label = np.asarray(label, dtype=np.int32)
+        print("batch_image.shape:{},batch_label:{}".format(image.shape, label))
+        image_utils.cv_show_image("image", image)