pybaseutils 2.1.4__tar.gz → 2.1.6__tar.gz

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybaseutils
-Version: 2.1.4
+Version: 2.1.6
 Summary: pybaseutils
 Home-page: https://github.com/PanJinquan/base-utils
 Author: PanJinquan

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/__init__.py

@@ -5,4 +5,4 @@
     @Date   : 2019-05-07 17:40:27
 """
 
-__version__ = '2.1.4'
+__version__ = '2.1.6'

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/audio/audio_utils.py

@@ -20,6 +20,7 @@ import matplotlib.pyplot as plt
 from scipy.fft import fft
 from IPython.display import Audio, display
 from playsound import playsound
+from scipy.io import wavfile
 
 import numpy as np
 
@@ -158,27 +159,56 @@ def save_pcm_from_numpy(audio_data: np.ndarray, pcm_file, data_type=np.int16):
     return bytes
 
 
-def read_audio(audio_file, sr=16000, mono=True):
+def librosa_load(audio_file, sr=None, mono=True):
     """
     默认将多声道音频文件转换为单声道，并返回一维数组；
     如果你需要处理多声道音频文件，可以使用 mono=False,参数来保留所有声道，并返回二维数组。
     :param audio_file:
-    :param sr: sampling rate
+    :param sr: sampling rate 16000
     :param mono: 设置为true是单通道，否则是双通道
     :return:
     """
-    audio_data, sr = librosa.load(audio_file, sr=sr, mono=mono)
+    audio_data, sr = librosa.load(audio_file, sr=sr, mono=mono)  # 非常耗时
     return audio_data, sr
 
 
-def read_audio_sf(audio_file, sr=16000):
+def soundfile_load(audio_file, sr=None, mono=True):
     """
     默认会将多声道音频文件的每个声道分别存储在二维数组的不同列中，因此返回的是一个二维数组
     :param audio_file:
+    :param sr: sampling rate 当设置的采样率与音频原始采样率不一致时，会进行重采样，导致非常耗时
+    :param mono: 设置为true是单通道，否则是双通道
+    :return:
+    """
+    audio_data, orig_sr = sf.read(audio_file, samplerate=None, dtype="float32")  # 不能直接修改samplerate
+    if mono and audio_data.ndim == 2:
+        audio_data = np.mean(audio_data, axis=-1)
+    if sr is None or orig_sr == sr:
+        sr = orig_sr
+    else:
+        audio_data = librosa.resample(y=audio_data, orig_sr=orig_sr, target_sr=sr)  # 使用librosa进行重采样至目标采样率
+    return audio_data, sr
+
+
+def wavfile_load(audio_file):
+    sr, wav_data = wavfile.read(audio_file)  # int16类型
+    # 转为float64类型
+    wav_data = wav_data / (32768)
+    # wav_data:[-0.03305054 -0.03561401 -0.038114697]
+    return wav_data, sr
+
+
+def read_audio(audio_file, sr=16000, mono=True):
+    """
+    默认将多声道音频文件转换为单声道，并返回一维数组；
+    如果你需要处理多声道音频文件，可以使用 mono=False,参数来保留所有声道，并返回二维数组。
+    :param audio_file:
     :param sr: sampling rate
+    :param mono: 设置为true是单通道，否则是双通道
     :return:
     """
-    audio_data, sr = sf.read(audio_file, samplerate=sr)
+    audio_data, sr = librosa_load(audio_file, sr=sr, mono=mono)  # 巨慢
+    # audio_data, sr = soundfile_load(audio_file, sr=sr, mono=mono)  # 巨慢
     return audio_data, sr
 
 
@@ -412,6 +442,16 @@ def display_freq_domain(audio, sr=16000):
     plt.show()
 
 
+def test_time(audio_file, sr=None):
+    from pybaseutils import time_utils
+    for i in range(10):
+        with time_utils.Performance("soundfile_load") as p:
+            audio_data1, sr1 = soundfile_load(audio_file, sr=sr)
+        with time_utils.Performance("librosa_load") as p:
+            audio_data2, sr2 = librosa_load(audio_file, sr=sr)
+    print(np.sum(np.abs(audio_data2 - audio_data1)))
+
+
 if __name__ == '__main__':
     # video_file = "../data/video/kunkun_cut.mp4"
     # audio_file = "../data/video/kunkun_cut.mp3"
@@ -420,8 +460,10 @@ if __name__ == '__main__':
     audio_file = "../../data/video/kunkun_cut.mp3"
     video_out = "../../data/video/test-video-result.mp4"
     audio_file = "../../data/audio/bus_chinese.wav"
+
+    test_time(audio_file)
     # playsound(audio_file)
     # playsound(audio_file)
-    display_time_domain(audio_file)
-    display_freq_domain(audio_file)
+    # display_time_domain(audio_file)
+    # display_freq_domain(audio_file)
     # merge_video_audio(video_file, audio_file, video_out)

pybaseutils-2.1.6/pybaseutils/cvutils/nms_utils.py

@@ -0,0 +1,62 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : PKing
+    @E-mail : 
+    @Date   : 2024-10-09 14:19:18
+    @Brief  :
+"""
+import cv2
+import numpy as np
+import torch
+import torchvision
+from pybaseutils import image_utils
+
+
+def nms_boxes_cv2(boxes: np.ndarray, scores: np.ndarray, labels: np.ndarray, score_threshold=0.5, nms_threshold=0.5,
+                  top_k=None, use_batch=True):
+    """
+    :param boxes:
+    :param scores:
+    :param labels:
+    :param score_threshold:
+    :param nms_threshold:
+    :param top_k:
+    :return:
+    """
+    if use_batch:
+        # TODO opencv4.7.0版本中增加了NMSBoxesBatched函数，可分类做nms
+        xywh = image_utils.xyxy2xywh(boxes)
+        index = cv2.dnn.NMSBoxesBatched(xywh.tolist(), scores.tolist(), labels.tolist(),
+                                        score_threshold=score_threshold, nms_threshold=nms_threshold, top_k=top_k)
+    else:
+        # TODO CV2 NMSBoxes没有区分类别
+        # 如果你不想每个类别都做一次nms,而是所有类别一起做nms
+        # 就需要把不同类别的目标框尽量没有重合，不至于把不同类别的IOU大的目标框滤掉
+        # 先用每个类别id乘一个很大的数，作为offset,把每个类别的box坐标都加上相应的offset,这是batched nms
+        max_wh = 7680
+        c = (labels * max_wh).reshape(-1, 1).astype(np.float32)  # classes
+        boxes_ = boxes + c  # boxes (offset by class)
+        xywh = image_utils.xyxy2xywh(boxes_)
+        index = cv2.dnn.NMSBoxes(xywh.tolist(), scores.tolist(), score_threshold=score_threshold,
+                                 nms_threshold=nms_threshold,top_k=top_k)
+    return index
+
+
+def nms_boxes_torch(boxes: np.ndarray, scores: np.ndarray, labels: np.ndarray, score_threshold=0.5, nms_threshold=0.5,
+                    top_k=None):
+    """
+    :param boxes:
+    :param scores:
+    :param labels:
+    :param score_threshold:
+    :param nms_threshold:
+    :param top_k:
+    :return:
+    """
+    max_wh = 7680
+    c = (labels * max_wh).reshape(-1, 1).astype(np.float32)  # classes
+    boxes_ = boxes + c  # boxes (offset by class)
+    index = torchvision.ops.nms(torch.from_numpy(boxes_),
+                                torch.from_numpy(scores),
+                                iou_threshold=nms_threshold)  # NMS
+    return index

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/dataloader/parser_image_folder.py

@@ -56,7 +56,8 @@ class FolderDataset(parser_image_text.TextDataset):
         for i, dir in enumerate(data_file):
             if not os.path.exists(dir): raise Exception("文件不存在，image_dir:{}".format(dir))
             paths, labels = file_utils.get_files_labels(dir, postfix=file_utils.IMG_POSTFIX)
-            print("loading data from:{},have {}".format(dir, len(paths)))
+            if len(paths) == 0: raise Exception("文件为空:{}".format(dir))
+            print("loading data from:{},have {},label:{}".format(dir, len(paths), list(set(labels))))
             # TODO # 避免多个数据集的相同的label
             if use_sub:  labels = [os.path.join(str(i), l) for l in labels]
             data = [{"file": p, "label": l} for p, l in zip(paths, labels)]
@@ -69,7 +70,7 @@ if __name__ == '__main__':
     from torchvision import transforms
 
     image_dir = ['/home/PKing/nasdata/release/infrastructure/DMClassification/data/dataset/train']
-    class_name = None
+    class_name = []
     input_size = [224, 224]
     rgb_mean = [0., 0., 0.]
     rgb_std = [1.0, 1.0, 1.0]

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/dataloader/parser_image_text.py

@@ -57,19 +57,18 @@ class TextDataset(Dataset):
             balance_nums = self.data_resample.balance_nums  # resample后，每个类别的分布
         self.class_count = self.count_class_info(self.item_list, class_name=self.class_name,
                                                  label_index=self.label_index)
-        self.num_images = len(self.item_list)
         self.classes = list(self.class_dict.values())
         self.num_classes = max(self.classes) + 1
+        self.num_samples = len(self.item_list)
         self.info(save_info=kwargs.get("save_info", ""))
 
     def info(self, save_info=""):
         print("----------------------- {} DATASET INFO -----------------------".format(self.phase.upper()))
-        print("Dataset have images   :{}".format(len(self.item_list)))
+        print("Dataset num_samples   :{}".format(len(self.item_list)))
+        print("Dataset num_classes   :{}".format(self.num_classes))
         print("Dataset class_name    :{}".format(self.class_name))
         print("Dataset class_dict    :{}".format(self.class_dict))
         print("Dataset class_count   :{}".format(self.class_count))
-        print("Dataset num images    :{}".format(len(self.item_list)))
-        print("Dataset num_classes   :{}".format(self.num_classes))
         print("Dataset resample      :{}".format(self.resample))
         if save_info:
             if not os.path.exists(save_info): os.makedirs(save_info)
@@ -88,7 +87,7 @@ class TextDataset(Dataset):
     def parser_dataset(self, data_file, data_root="", label_index="label", shuffle=False, check=False):
         """
         保存格式：[path,label] 或者 [path,label,xmin,ymin,xmax,ymax]
-        :param data_file:
+        :param data_file: List([])
         :param data_root:
         :param label_index: label index
         :param shuffle:
@@ -164,7 +163,7 @@ class TextDataset(Dataset):
                 print("no file:{}".format(file))
                 continue
             dst_list.append(item)
-        print("have nums image:{},legal image:{}".format(len(item_list), len(dst_list)))
+        print("have nums samples:{},legal samples:{}".format(len(item_list), len(dst_list)))
         return dst_list
 
     def __getitem__(self, index):
@@ -180,7 +179,7 @@ class TextDataset(Dataset):
             image = Image.fromarray(image)
             image = self.transform(image)
         if image is None:
-            index = int(random.uniform(0, self.num_images))
+            index = int(random.uniform(0, self.num_samples))
             return self.__getitem__(index)
         return {"image": image, "label": label, "file": file}
 

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/dataloader/parser_labelme.py

@@ -41,10 +41,10 @@ class LabelMeDataset(Dataset):
         :param data_root:
         :param anno_dir:
         :param image_dir:
-        :param class_name: 当为name时，会获得由于类别
+        :param class_name: 当class_name=None且check=True,将自动获取所有class,当class_name=[]会直接返回name
         :param use_rgb:
         :param shuffle:
-        :param check:
+        :param check: 当class_name=None且check=True,将自动获取所有class
         :param min_points: 当标注的轮廓点的个数小于min_points，会被剔除；负数不剔除
         :param kwargs:
         """

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/image_utils.py

@@ -1728,8 +1728,8 @@ def save_image(image_file, image, uint8=False, use_rgb=False):
     cv2.imwrite(image_file, image)
 
 
-def nms_boxes_cv2(boxes: np.ndarray, scores: np.ndarray, labels: np.ndarray, image_size=(),
-                  score_threshold=0.5, nms_threshold=0.45, eta=None, top_k=None):
+def nms_boxes_cv2(boxes: np.ndarray, scores: np.ndarray, labels: np.ndarray, score_threshold=0.5, nms_threshold=0.45,
+                  eta=None, top_k=None, use_class=True):
     """
     NMS
     fix a bug: cv2.dnn.NMSBoxe bboxes, scores params must be list and float data,can not be float32 or int
@@ -1741,17 +1741,18 @@ def nms_boxes_cv2(boxes: np.ndarray, scores: np.ndarray, labels: np.ndarray, ima
     :param nms_threshold:
     :return:
     """
-    if isinstance(boxes, np.ndarray): boxes = boxes.tolist()
-    if isinstance(scores, np.ndarray): scores = scores.tolist()
-    indices = cv2.dnn.NMSBoxes(boxes, scores, score_threshold=score_threshold, nms_threshold=nms_threshold,
-                               eta=eta, top_k=top_k)
-    if isinstance(boxes, list): boxes = np.asarray(boxes)
-    if isinstance(scores, list): scores = np.asarray(scores)
-    if isinstance(labels, list): labels = np.asarray(labels)
-    dst_boxes = boxes[indices]
-    dst_score = scores[indices]
-    dst_label = labels[indices]
-    return dst_boxes, dst_score, dst_label
+    if use_class:
+        # TODO opencv4.7.0版本中增加了NMSBoxesBatched函数，可分类做nms
+        index = cv2.dnn.NMSBoxesBatched(boxes.tolist(), scores.tolist(), labels.tolist(),
+                                        score_threshold=score_threshold, nms_threshold=nms_threshold,
+                                        eta=eta, top_k=top_k)
+    else:
+        # TODO NMSBoxes没有区分类别
+        index = cv2.dnn.NMSBoxes(boxes.tolist(), scores.tolist(), score_threshold=score_threshold,
+                                 nms_threshold=nms_threshold, eta=eta, top_k=top_k)
+    return index
+
+
 
 
 def file2base64(file):

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils/pandas_utils.py

@@ -74,6 +74,19 @@ def construct_pd(index, columns_name, content, filename=None):
     return df
 
 
+def dict2pd(data: dict, T=False):
+    """
+    :param data:
+    :param T:
+    :return:
+    """
+    if T:
+        df = pd.DataFrame.from_dict(data)  # 键按照列进行转换
+    else:
+        df = pd.DataFrame.from_dict(data, orient='index')  # 键按照行进行转换
+    return df
+
+
 if __name__ == "__main__":
     class_name = ['C1', 'C2', 'C3']
     labels = [100, 200, 300]
@@ -85,3 +98,11 @@ if __name__ == "__main__":
     filename = "my_test.csv"
     df = construct_pd(index, columns_name, content, filename)
     print(df)
+
+    data = {
+        'name1': ["A0", "A1", "A2"],
+        'name2': ["B0", "B1", "B2"],
+        'name3': ["C0", "C1", "C2"]
+    }
+    df = dict2pd(data)
+    save_csv("data.csv", df)

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybaseutils
-Version: 2.1.4
+Version: 2.1.6
 Summary: pybaseutils
 Home-page: https://github.com/PanJinquan/base-utils
 Author: PanJinquan

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/pybaseutils.egg-info/SOURCES.txt

@@ -66,6 +66,7 @@ pybaseutils/cvutils/__init__.py
 pybaseutils/cvutils/corner_utils.py
 pybaseutils/cvutils/monitor.py
 pybaseutils/cvutils/mouse_utils.py
+pybaseutils/cvutils/nms_utils.py
 pybaseutils/cvutils/video_utils.py
 pybaseutils/dataloader/__init__.py
 pybaseutils/dataloader/balanced_classes.py
@@ -174,6 +175,7 @@ test_py/aije/demo_labelme_crop.py
 test_py/aije/demo_video_aije.py
 test_py/aije/demo_voc_crop.py
 test_py/aije/demo_voc_vis.py
+test_py/aije/get_pair_data.py
 test_py/aije/video_convertor.py
 test_py/audio/__init__.py
 test_py/audio/demo.py
@@ -238,6 +240,9 @@ test_py/image_correction/demo_correction_v2.py
 test_py/image_correction/demo_correction_v3.py
 test_py/pose/__init__.py
 test_py/pose/human_pose.py
+test_py/redis_py/__init__.py
+test_py/redis_py/knn_search.py
+test_py/redis_py/redis_client.py
 test_py/registry/__init__.py
 test_py/registry/base.py
 test_py/registry/component.py

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/test_py/aije/demo_labelme_crop.py

@@ -64,7 +64,7 @@ if __name__ == "__main__":
           '绝缘手套', '绝缘鞋', '脚扣', '螺丝', '身穿工作服', '遮拦杆', '铁架', '铝扎线']
     室外: []
     """
-    anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-v2-det/dataset-v25/json"
+    anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-v2-det/dataset-v21/json"
     class_name = None  # 室内\
     out_dir = os.path.join(os.path.dirname(anno_dir), "crops")
     dataset = parser_labelme.LabelMeDatasets(filename=None,

pybaseutils-2.1.6/test_py/aije/get_pair_data.py

@@ -0,0 +1,18 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : PKing
+    @E-mail : 
+    @Date   : 2024-08-24 11:05:52
+    @Brief  :
+"""
+import os
+import numpy as np
+import itertools
+from pybaseutils import file_utils, image_utils
+
+if __name__ == '__main__':
+    image_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-action-cvlm-v2/train-v2/dataset/all_action"
+    pairs_file = os.path.join(image_dir, "pairs.txt")
+    # pairs = file_utils.get_pair_data(image_dir, pair_num=-1)
+    pairs = file_utils.get_pair_data(image_dir, pair_num=10000)
+    file_utils.write_data(pairs_file, pairs)

{pybaseutils-2.1.4 → pybaseutils-2.1.6}/test_py/demo_lableme_vis.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
     # anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-outdoor-det-fix/dataset-v01/json"
     # anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-outdoor-det-fix/dataset-v23/json"
     anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-indoor-det/dataset-test/json"
-    anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-v2-det/dataset-v24/json"
+    anno_dir = "/home/PKing/nasdata/dataset-dmai/AIJE/dataset/aije-action-cvlm-v2/第3批数据/json"
     names = None
     # names = ['身穿工作服,未穿工作服', '手,手穿绝缘手套,手穿棉纱手套,手穿其他手套']
     dataset = parser_labelme.LabelMeDatasets(filename=None,

pybaseutils-2.1.6/test_py/redis_py/__init__.py

@@ -0,0 +1,7 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : PKing
+    @E-mail : 
+    @Date   : 2024-10-09 19:53:29
+    @Brief  :
+"""

pybaseutils-2.1.6/test_py/redis_py/knn_search.py

@@ -0,0 +1,14 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : PKing
+    @E-mail : 
+    @Date   : 2024-10-09 19:55:10
+    @Brief  :
+"""
+from test_py.redis_py.redis_client import r_client, CacheMethod
+from redisearch import Client, Query
+
+if __name__ == "__main__":
+    CacheMethod.set("name", "test")
+    keys = CacheMethod().get_all_keys()
+    print(keys)

pybaseutils-2.1.6/test_py/redis_py/redis_client.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+redis client
+"""
+import redis
+from redis import StrictRedis
+
+REDIS_URI = 'redis://@10.12.51.100:6379'
+pool = redis.ConnectionPool().from_url(REDIS_URI)
+r_client = StrictRedis(connection_pool=pool)
+
+
+class CacheMethod:
+    """redis method"""
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_client():
+        return r_client
+
+    @staticmethod
+    def set(key, value):
+        return r_client.set(key, value)
+
+    @staticmethod
+    def get(key):
+        return r_client.get(key)
+
+    @staticmethod
+    def exists(key):
+        return r_client.exists(key)
+
+    @staticmethod
+    def delete(key):
+        return r_client.delete(key)
+
+    @staticmethod
+    def keys(key):
+        return r_client.keys(key)
+
+    @staticmethod
+    def incr(key):
+        return r_client.incr(key)
+
+    @staticmethod
+    def decr(key):
+        return r_client.decr(key)
+
+    @staticmethod
+    def expire(key, time):
+        """设置，过期删除时间，必须创建后再设置"""
+        return r_client.expire(key, time)
+
+    @staticmethod
+    def get_all_keys():
+        """获得所有keys"""
+        keys = r_client.keys('*')
+        keys = keys if keys else []
+        keys = [k.decode('utf-8') for k in keys]
+        return keys
+
+    @staticmethod
+    def del_all_keys(ignore=[]):
+        """删除所有keys"""
+        keys = CacheMethod.get_all_keys()
+        try:
+            for k in keys:
+                if k in ignore: continue
+                CacheMethod.delete(k)
+                print("INFO: Redis delete key:{}".format(k))
+        except Exception as e:
+            print(e)
+            print("Error: Redis delete key:{}".format(k))
+
+    @staticmethod
+    def set_all_expire(time, ignore=[]):
+        """设置所有keys的过期时间"""
+        keys = CacheMethod.get_all_keys()
+        try:
+            for k in keys:
+                if k in ignore: continue
+                CacheMethod.expire(k, time=time)
+                print("INFO: Redis set key={}, expire={}s".format(k, time))
+        except Exception as e:
+            print(e)