pybaseutils 2.2.9__tar.gz → 2.2.10__tar.gz

{pybaseutils-2.2.9 → pybaseutils-2.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybaseutils
-Version: 2.2.9
+Version: 2.2.10
 Summary: pybaseutils
 Home-page: https://github.com/PanJinquan/base-utils
 Author: PanJinquan

{pybaseutils-2.2.9 → pybaseutils-2.2.10}/pybaseutils/__init__.py

@@ -5,4 +5,4 @@
     @Date   : 2019-05-07 17:40:27
 """
 
-__version__ = '2.2.9'
+__version__ = '2.2.10'

pybaseutils-2.2.10/pybaseutils/http_utils.py

@@ -0,0 +1,90 @@
+# -*- coding:utf-8 -*-
+import logging
+import time
+import requests
+import json
+from pybaseutils import log_utils
+
+logger = log_utils.get_logger()
+
+
+def post(url, params, timeout=6, max_retries=1, **kwargs):
+    """
+    :param url: 请求URL
+    :param params: 请求参数
+    :param timeout: 设置超时
+    :return:
+    """
+    result = None
+    counts = 0
+    while counts < max_retries:
+        try:
+            t1 = time.time()
+            # r = requests.get(url, params=params, timeout=timeout,**kwargs)
+            r = requests.post(url, json=params, timeout=timeout, **kwargs)
+            t2 = time.time()
+            elapsed = (t2 - t1) * 1000
+            code = r.status_code
+            if code == 200:
+                logger.info(f'code={code}, url={url}, elapsed:{elapsed:3.3f}ms')
+                result = r.json()
+                break
+            else:
+                r.raise_for_status()  # 如果响应状态码不是200，抛出异常
+        except Exception as e:
+            counts += 1
+            logger.error(f'Error msg:{e}, try to retry times={counts}/{max_retries}')
+            time.sleep(0.1)
+    return result
+
+
+def get(url, params, timeout=6, max_retries=1, **kwargs):
+    """
+    :param url: 请求URL
+    :param params: 请求参数
+    :param timeout: 设置超时
+    :return:
+    """
+    result = None
+    counts = 0
+    while counts < max_retries:
+        try:
+            t1 = time.time()
+            r = requests.get(url, params=params, timeout=timeout, **kwargs)
+            # r = requests.post(url, json=params, timeout=timeout, **kwargs)
+            t2 = time.time()
+            elapsed = (t2 - t1) * 1000
+            code = r.status_code
+            if code == 200:
+                logger.info(f'code={code}, url={url}, elapsed:{elapsed:3.3f}ms')
+                result = r.json()
+                break
+            else:
+                r.raise_for_status()  # 如果响应状态码不是200，抛出异常
+        except Exception as e:
+            counts += 1
+            logger.error(f'Error msg:{e}, try to retry times={counts}/{max_retries}')
+            time.sleep(0.1)
+    return result
+
+
+def tojson(data: dict, keys: list):
+    """
+    指定需要反序列的数据
+    :param data:
+    :param keys:
+    :return:
+    """
+    if isinstance(data, dict):
+        for k, v in data.items():
+            if k in keys:
+                try:
+                    data[k] = json.loads(v)
+                except Exception as e:
+                    print(e)
+            else:
+                data[k] = tojson(v, keys=keys)
+    elif isinstance(data, list):
+        for i in range(len(data)):
+            data[i] = tojson(data[i], keys=keys)
+    return data

{pybaseutils-2.2.9 → pybaseutils-2.2.10}/pybaseutils/image_utils.py

@@ -166,25 +166,6 @@ def check_point(point):
     return r
 
 
-def transpose(data):
-    data = data.transpose(2, 0, 1)  # HWC->CHW
-    return data
-
-
-def untranspose(data):
-    if len(data.shape) == 3:
-        data = data.transpose(1, 2, 0).copy()  # 通道由[c,h,w]->[h,w,c]
-    else:
-        data = data.transpose(1, 0).copy()
-    return data
-
-
-def swap_image(image):
-    # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-    image = image[:, :, ::-1]  # RGB->BGR
-    return image
-
-
 def show_batch_image(title, batch_images, index=0):
     '''
     批量显示图片
@@ -332,6 +313,31 @@ def get_prewhiten_image(x):
     return y
 
 
+def transpose(data):
+    data = data.transpose(2, 0, 1)  # HWC->CHW
+    return data
+
+
+image_hwc2chw = transpose
+
+
+def untranspose(data):
+    if len(data.shape) == 3:
+        data = data.transpose(1, 2, 0).copy()  # 通道由[c,h,w]->[h,w,c]
+    else:
+        data = data.transpose(1, 0).copy()
+    return data
+
+
+image_chw2hwc = untranspose
+
+
+def swap_image(image):
+    # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    image = image[:, :, ::-1]  # RGB->BGR
+    return image
+
+
 def image_normalize(image, mean=None, std=None):
     '''
     正则化，归一化
@@ -357,9 +363,9 @@ def image_normalize(image, mean=None, std=None):
 
 def image_unnormalize(image, mean=None, std=None):
     """
-    正则化，归一化
+    反归一化
     image = image/255.0
-    image[channel] = (image[channel] - mean[channel]) / std[channel]
+    image[channel] = image[channel] * std[channel]+ mean[channel])
     :param image: numpy image
     :param mean: [0.5,0.5,0.5]
     :param std:  [0.5,0.5,0.5]

pybaseutils-2.2.10/pybaseutils/log_utils.py

@@ -0,0 +1,64 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : PKing
+    @E-mail :
+    @Date   : 2024-12-10 22:22:58
+    @Brief  :
+"""
+import os
+import sys
+import logging
+from logging.handlers import TimedRotatingFileHandler
+
+
+def get_logger(name='APPLOG', level="info", logfile=None, is_main_process=True):
+    """
+    :param name:
+    :param logfile:
+    :param level: CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET
+    :return:
+    """
+    # 创建logger对象
+    if is_main_process:
+        logger = logging.getLogger(name)
+        # 如果logger已经有，则不添加
+        if logger.handlers: return logger
+        logger.setLevel(level.upper())
+        # 创建格式化器
+        formatter = logging.Formatter(
+            fmt="%(asctime)s|%(levelname)6s|%(filename)20s|%(funcName)20s|Line%(lineno)4s|%(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S"
+        )
+        # 创建控制台处理器
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+        # 创建日志目录
+        if logfile:
+            os.makedirs(os.path.dirname(logfile), exist_ok=True)
+            # 创建文件处理器（按时间轮转）
+            file_handler = TimedRotatingFileHandler(
+                logfile,
+                when='midnight',  # 每天午夜切换新文件
+                interval=1,  # 间隔为1天
+                backupCount=30,  # 保留30天的日志文件
+                encoding='utf-8'
+            )
+            file_handler.setFormatter(formatter)
+            logger.addHandler(file_handler)
+    else:
+        logger = logging.getLogger(name)
+        logger.setLevel("warning")
+    return logger
+
+
+if __name__ == '__main__':
+    logger = get_logger(name='APPLOG', level="debug", logfile="./log.log")
+    # 测试日志输出
+    logger.debug('-----------------')
+    logger.debug('这是一条调试信息')
+    logger.info('这是一条信息')
+    logger.warning('这是一条警告')
+    logger.error('这是一条错误信息')
+    logger.critical('这是一条严重错误信息')

{pybaseutils-2.2.9 → pybaseutils-2.2.10}/pybaseutils/text_utils.py

@@ -32,6 +32,7 @@ def find_match_texts(texts: list, pattern: list, org=True):
     :param org:  True,返回匹配成功的原始字符串，False 返回匹配成功的子串
     :return:
     """
+    if isinstance(pattern, str): pattern = [pattern]
     out = []
     for text in texts:
         for sub in pattern:

{pybaseutils-2.2.9 → pybaseutils-2.2.10}/pybaseutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybaseutils
-Version: 2.2.9
+Version: 2.2.10
 Summary: pybaseutils
 Home-page: https://github.com/PanJinquan/base-utils
 Author: PanJinquan

{pybaseutils-2.2.9 → pybaseutils-2.2.10}/pybaseutils.egg-info/SOURCES.txt

@@ -11,11 +11,11 @@ pybaseutils/file_utils.py
 pybaseutils/font_utils.py
 pybaseutils/geometry_tools.py
 pybaseutils/heatmap_utils.py
+pybaseutils/http_utils.py
 pybaseutils/image_utils.py
 pybaseutils/json_utils.py
 pybaseutils/log.py
 pybaseutils/log_utils.py
-pybaseutils/logger.py
 pybaseutils/numpy_utils.py
 pybaseutils/pandas_utils.py
 pybaseutils/plot_utils.py

pybaseutils-2.2.10/test_py/demo3.py

@@ -0,0 +1,98 @@
+# -*-coding: utf-8 -*-
+"""
+    @Author : panjq
+    @E-mail : pan_jinquan@163.com
+    @Date   : 2022-08-30 09:45:44
+    @Brief  :
+"""
+
+import os
+import numpy as np
+import cv2
+from tqdm import tqdm
+from pybaseutils import image_utils, file_utils
+from pybaseutils.converter import build_voc
+
+
+def save_voc_dataset(bboxes, labels, image_file, image_shape, out_xml_dir):
+    """
+    保存VOC数据集
+    :param bboxes:
+    :param labels:
+    :param image_file:
+    :param image_shape:
+    :param out_xml_dir:
+    :return:
+    """
+    basename = os.path.basename(image_file)
+    image_id = basename.split(".")[0]
+    objects = []
+    for box, name in zip(bboxes, labels):
+        objects.append({"name": name, "bndbox": box})
+    xml_path = file_utils.create_dir(out_xml_dir, None, "{}.xml".format(image_id))
+    build_voc.write_voc_xml_objects(basename, image_shape, objects, xml_path)
+
+
+def save_crop_dataset(image, bboxes, labels, image_file, crop_root):
+    """
+    裁剪检测区域
+    :param image:
+    :param bboxes:
+    :param labels:
+    :param image_file:
+    :param crop_root:
+    :return:
+    """
+    basename = os.path.basename(image_file)
+    image_id = basename.split(".")[0]
+    crops = image_utils.get_bboxes_image(image, bboxes, size=None)
+    for i, (img, label) in enumerate(zip(crops, labels)):
+        file = file_utils.create_dir(crop_root, label, "{}_{:0=3d}.jpg".format(image_id, i))
+        cv2.imwrite(file, img)
+
+
+def convert_HaGRID_dataset(data_root, vis=True):
+    """
+    将HaGRID转换为VOC和分类数据集
+    :param data_root: HaGRID数据集个根目录
+    :param vis: 是否可视化效果
+    :return:
+    """
+    sub_list = file_utils.get_sub_paths(data_root)
+    class_names = []
+    for sub in sub_list:
+        anno_file = os.path.join(data_root, sub, "{}.json".format(sub))
+        annotation = file_utils.read_json_data(anno_file)
+        image_list = file_utils.get_images_list(os.path.join(data_root, sub, "JPEGImages"))
+        print("process:{},nums:{}".format(anno_file, len(image_list)))
+        # 保存VOC格式的xml文件
+        out_xml_dir = os.path.join(data_root, sub, "Annotations")
+        # 裁剪并保存标注框区域的图片
+        out_crop_dir = os.path.join(data_root, sub, "Classification")
+        for image_file in tqdm(image_list):
+            basename = os.path.basename(image_file)
+            image_id = basename.split(".")[0]
+            image = cv2.imread(image_file)
+            anno = annotation[image_id]
+            h, w = image.shape[:2]
+            # [top left X pos, top left Y pos, width, height]
+            bboxes = image_utils.rects2bboxes(anno['bboxes'])
+            bboxes = np.asarray(bboxes) * [w, h, w, h]
+            labels = anno['labels']
+            class_names += labels
+            image_shape = image.shape
+            assert len(bboxes) == len(labels)
+            if out_xml_dir:
+                save_voc_dataset(bboxes, labels, image_file, image_shape, out_xml_dir)
+            if out_crop_dir:
+                save_crop_dataset(image, bboxes, labels, image_file, out_crop_dir)
+            if vis:
+                image = image_utils.draw_image_bboxes_text(image, bboxes, labels, color=(255, 0, 0))
+                image_utils.cv_show_image("image", image, use_rgb=False)
+    class_names = set(class_names)
+    print(class_names)
+
+
+if __name__ == "__main__":
+    data_root = "path/to/gesture/HaGRID/trainval"
+    convert_HaGRID_dataset(data_root, vis=True)

pybaseutils-2.2.9/pybaseutils/log_utils.py

@@ -1,191 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-# --------------------------------------------------------
-# @Author : Pan
-# @Date   : 2019-9-20 13:18:34
-# --------------------------------------------------------
-"""
-import os
-import datetime
-import logging
-import threading
-import re
-import time
-from logging.handlers import TimedRotatingFileHandler
-
-
-# from memory_profiler import profile
-
-
-def singleton(cls):
-    _instance_lock = threading.Lock()
-    instances = {}
-
-    def _singleton(*args, **kwargs):
-        with _instance_lock:
-            if cls not in instances:
-                instances[cls] = cls(*args, **kwargs)
-            return instances[cls]
-
-    return _singleton
-
-
-@singleton  # 使用singleton，会出现loger的level失效的问题
-class CustomLogger(logging.Logger):
-    def __init__(self, name="LOG", level="debug"):
-        """
-        Initialize the logger with a name and an optional level.
-        Args:
-            name:
-            level: debug,info,warning,critical,fatal
-        """
-        super().__init__(name)
-        # super(CustomLogger, self).__init__(name)
-        self.setLevel(level=level)
-
-    @staticmethod
-    def levels(level):
-        if level == 'debug':
-            return logging.DEBUG
-        if level == 'info':
-            return logging.INFO
-        if level == 'warning':
-            return logging.WARN
-        if level == 'critical':
-            return logging.CRITICAL
-        if level == 'fatal':
-            return logging.FATAL
-        return logging.DEBUG
-
-    def setLevel(self, level):
-        """
-        Args:
-            level: debug,info,warning,critical,fatal
-        Returns:
-        """
-        level = self.levels(level)
-        super().setLevel(level)
-
-    @staticmethod
-    def set_format(handler, format):
-        # handler.suffix = "%Y%m%d"
-        if format:
-            logFormatter = logging.Formatter("%(asctime)s %(filename)s %(funcName)s %(levelname)s: %(message)s",
-                                             "%Y-%m-%d %H:%M:%S")
-        else:
-            logFormatter = logging.Formatter("%(levelname)s: %(message)s")
-        handler.setFormatter(logFormatter)
-
-    def show_batch_tensor(self, title, batch_imgs, index=0):
-        pass
-
-
-class FileHandler(TimedRotatingFileHandler):
-    def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False,
-                 atTime=None):
-        logging.handlers.BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
-        self.when = when.upper()
-        self.backupCount = backupCount
-        self.utc = utc
-        self.atTime = atTime
-        if self.when == 'S':
-            self.interval = 1  # one second
-            self.suffix = "%Y-%m-%d_%H-%M-%S"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$"
-        elif self.when == 'M':
-            self.interval = 60  # one minute
-            self.suffix = "%Y-%m-%d_%H-%M"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.\w+)?$"
-        elif self.when == 'H':
-            self.interval = 60 * 60  # one hour
-            self.suffix = "%Y-%m-%d_%H"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.\w+)?$"
-        elif self.when == 'D' or self.when == 'MIDNIGHT':
-            self.interval = 60 * 60 * 24  # one day
-            self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
-        elif self.when.startswith('W'):
-            self.interval = 60 * 60 * 24 * 7  # one week
-            if len(self.when) != 2:
-                raise ValueError("You must specify a day for weekly rollover from 0 to 6 (0 is Monday): %s" % self.when)
-            if self.when[1] < '0' or self.when[1] > '6':
-                raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
-            self.dayOfWeek = int(self.when[1])
-            self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
-        elif self.when == 'Y':
-            self.interval = 60 * 60 * 24 * 365  # one yes
-            self.suffix = "%Y-%m-%d"
-            self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
-        else:
-            raise ValueError("Invalid rollover interval specified: %s" % self.when)
-
-        self.extMatch = re.compile(self.extMatch, re.ASCII)
-        self.interval = self.interval * interval  # multiply by units requested
-        # The following line added because the filename passed in could be a
-        # path object (see Issue #27493), but self.baseFilename will be a string
-        filename = self.baseFilename
-        if os.path.exists(filename):
-            t = os.stat(filename)[logging.handlers.ST_MTIME]
-        else:
-            t = int(time.time())
-        self.rolloverAt = self.computeRollover(t)
-
-
-def set_logger(name="LOG", level="debug", logfile=None, format=False, is_main_process=True):
-    """
-    logger = set_logging(name="LOG", level="debug", logfile="log.txt", format=False)
-    url:https://cuiqingcai.com/6080.html
-    level级别：debug>info>warning>error>critical
-    :param level: 设置log输出级别
-    :param logfile: log保存路径，如果为None，则在控制台打印log
-    :param is_main_process: 是否是主进程
-    :return:
-    """
-    if not is_main_process:
-        level = "fatal"
-        logfile = None
-    # logger = logging.getLogger(name)
-    logger = CustomLogger(name, level=level)
-    if logfile and os.path.exists(logfile):
-        os.remove(logfile)
-    # define a FileHandler write messages to file
-    if logfile:
-        # filehandler = logging.handlers.RotatingFileHandler(filename="./log.txt")
-        # filehandler = TimedRotatingFileHandler(logfile, when="midnight", interval=1)
-        filehandler = FileHandler(logfile, when="Y", interval=1)
-        logger.set_format(filehandler, format)
-        logger.addHandler(filehandler)
-
-    # define a StreamHandler print messages to console
-    console = logging.StreamHandler()
-    logger.set_format(console, format)
-    logger.addHandler(console)
-    return logger
-
-
-def print_args(args):
-    logger = get_logger()
-    logger.info("---" * 10)
-    args = args.__dict__
-    for k, v in args.items():
-        # print("{}: {}".format(k, v))
-        logger.info("{}: {}".format(k, v))
-    logger.info("---" * 10)
-
-
-def get_logger(name="LOG", level="debug"):
-    logger = CustomLogger(name)
-    # if logger.isEnabledFor(CustomLogger.levels(level)):
-    #     logger = CustomLogger(name, level=level)
-    return logger
-
-
-if __name__ == '__main__':
-    logger = set_logger(logfile=None, level="debug")
-    logger1 = get_logger()
-    logger1.info("---" * 20)
-    logger1.debug("work_space:{}".format("work_dir"))
-    logger1.info("work_space:{}".format("work_dir"))
-    logger1.error("work_space:{}".format("work_dir"))
-    logger1.fatal("work_space:{}".format("work_dir"))

pybaseutils-2.2.9/pybaseutils/logger.py

@@ -1,62 +0,0 @@
-import os
-import logging
-import threading
-import sys
-import time
-
-local = threading.local()
-operateid_key = "operateid"
-
-
-def set_operate_id(val):
-    local.__setattr__(operateid_key, val)
-
-
-class ThreadLocalFormatter(logging.Formatter):
-
-    def format(self, record):
-        msg = super(ThreadLocalFormatter, self).format(record)
-        try:
-            opid = local.__getattribute__(operateid_key)
-            msg = opid + " " + msg
-        except Exception:
-            pass
-
-        return msg
-
-
-log_level = os.getenv("LOG_LEVEL", "info").upper()
-# LOG_FORMAT = "%(asctime)s %(levelname)s - %(filename)s[:%(lineno)d] - %(message)s"
-LOG_FORMAT = "%(asctime)s %(levelname)s %(message)s"
-logging.Formatter.default_msec_format = '%s.%03d'
-logging.Formatter.default_time_format = '%Y-%m-%dT%H:%M:%S'
-
-stream_handler = logging.StreamHandler()
-stream_handler.setFormatter(ThreadLocalFormatter(fmt=LOG_FORMAT))
-# 直接设置datefmt毫秒时间戳无效 有疑问请查看logging.Formatter代码 不推荐使用handlers配置
-# logging.basicConfig(level=log_level, format=LOG_FORMAT, handlers=[stream_handler])
-logging.basicConfig(level=log_level, format=LOG_FORMAT)
-
-# 重置handlers
-for h in logging.root.handlers:
-    logging.root.removeHandler(h)
-logging.root.addHandler(stream_handler)
-
-log = logging.getLogger("EP")
-
-
-def run_time_decorator(title=""):
-    def decorator(func):
-        def wrapper(*args, **kwargs):
-            # torch.cuda.synchronize()
-            t0 = time.time()
-            result = func(*args, **kwargs)
-            # torch.cuda.synchronize()
-            t1 = time.time()
-            # print("{} call {} elapsed: {}ms ".format(title, func.__name__, (t1 - t0) * 1000))
-            log.info("{}\t call {} \t elapsed:{:4.3f}ms \t".format(title, func.__name__, (t1 - t0) * 1000))
-            return result
-
-        return wrapper
-
-    return decorator

pybaseutils-2.2.9/test_py/demo3.py

@@ -1,16 +0,0 @@
-# -*-coding: utf-8 -*-
-"""
-    @Author : PKing
-    @E-mail : 390737991@qq.com
-    @Date   : 2022-12-31 11:37:30
-    @Brief  :
-"""
-
-import numpy as np
-import xmltodict
-from pybaseutils import image_utils
-import cv2
-
-if __name__ == '__main__':
-    """
-    """