xtn-tools-pro 1.0.0.0.4__py3-none-any.whl → 1.0.0.0.5__py3-none-any.whl

xtn_tools_pro/db/MysqlDB.py

@@ -0,0 +1,397 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/5/11    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import json
+import pymysql
+import datetime
+from pymysql import err
+from urllib import parse
+from pymysql import cursors
+from typing import List, Dict
+from dbutils.pooled_db import PooledDB
+from xtn_tools_pro.utils.log import Log
+from xtn_tools_pro.utils.sql import get_insert_sql, get_insert_batch_sql, get_update_sql
+
+log = Log(name="MysqlDB", color=True)
+
+
+def auto_retry(func):
+    def wapper(*args, **kwargs):
+        for i in range(3):
+            try:
+                return func(*args, **kwargs)
+            except (err.InterfaceError, err.OperationalError) as e:
+                log.error(
+                    """
+                    error:%s
+                    sql:  %s
+                    """
+                    % (e, kwargs.get("sql") or args[1])
+                )
+
+    return wapper
+
+
+class MysqlDBPro:
+    def __init__(self, ip, port, db, user_name, user_pass, **kwargs):
+        try:
+            self.connect_pool = PooledDB(
+                creator=pymysql,  # 指定数据库连接的创建方法，这里使用的是pymysql作为创建方法
+                mincached=1,  # 连接池中空闲连接的初始数量
+                maxcached=100,  # 连接池中空闲连接的最大数量
+                maxconnections=100,  # 连接池允许的最大连接数
+                blocking=True,  # 当连接池达到最大连接数时，是否阻塞等待连接释放
+                ping=7,  # 连接池中的连接在重新使用之前，是否需要进行ping操作来验证连接的有效性，这里的7是一个时间间隔，表示每隔7秒会对连接进行一次ping操作
+                host=ip,  # 数据库主机的IP地址
+                port=port,  # 数据库的端口号
+                user=user_name,  # 连接数据库的用户名
+                passwd=user_pass,  # 连接数据库的密码
+                db=db,  # 连接的数据库名
+                charset="utf8mb4",  # 连接数据库时使用的字符编码
+                cursorclass=cursors.SSCursor,  # 指定使用的游标类，这里使用的是cursors.SSCursor，该游标类在多线程下大批量插入数据时可以减少内存的使用
+            )  # cursorclass 使用服务的游标，默认的在多线程下大批量插入数据会使内存递增
+
+        except Exception as e:
+            log.error(
+                """
+            连接失败：
+            ip: {}
+            port: {}
+            db: {}
+            user_name: {}
+            user_pass: {}
+            exception: {}
+            """.format(
+                    ip, port, db, user_name, user_pass, e
+                )
+            )
+        else:
+            log.debug("连接到mysql数据库 %s : %s" % (ip, db))
+
+    @classmethod
+    def from_url(cls, url, **kwargs):
+        """
+
+        Args:
+            url: mysql://username:password@ip:port/db?charset=utf8mb4
+            url: mysql://username:password@127.0.0.1:port/db?charset=utf8mb4
+            **kwargs:
+
+        Returns:
+
+        """
+        url_parsed = parse.urlparse(url)
+
+        db_type = url_parsed.scheme.strip()
+        if db_type != "mysql":
+            raise Exception(
+                "url error, expect mysql://username:ip:port/db?charset=utf8mb4, but get {}".format(
+                    url
+                )
+            )
+
+        connect_params = {
+            "ip": url_parsed.hostname.strip(),
+            "port": url_parsed.port,
+            "user_name": url_parsed.username.strip(),
+            "user_pass": url_parsed.password.strip(),
+            "db": url_parsed.path.strip("/").strip(),
+        }
+
+        connect_params.update(kwargs)
+
+        return cls(**connect_params)
+
+    @staticmethod
+    def unescape_string(value):
+        if not isinstance(value, str):
+            return value
+        value = value.replace("\\0", "\0")
+        value = value.replace("\\\\", "\\")
+        value = value.replace("\\n", "\n")
+        value = value.replace("\\r", "\r")
+        value = value.replace("\\Z", "\032")
+        value = value.replace('\\"', '"')
+        value = value.replace("\\'", "'")
+        return value
+
+    def get_connection(self):
+        conn = self.connect_pool.connection(shareable=False)
+        # cursor = conn.cursor(cursors.SSCursor)
+        cursor = conn.cursor()
+
+        return conn, cursor
+
+    def close_connection(self, conn, cursor):
+        """
+            关闭数据库连接和游标对象
+        :param conn:
+        :param cursor:
+        :return:
+        """
+        if conn:
+            conn.close()
+        if cursor:
+            cursor.close()
+
+    def execute(self, sql):
+        """
+            执行sql
+        :param sql:
+        :return:
+        """
+        conn, cursor = None, None
+        try:
+            conn, cursor = self.get_connection()
+            cursor.execute(sql)
+            conn.commit()
+        except Exception as e:
+            log.error(
+                """
+                error:%s
+                sql:  %s
+            """
+                % (e, sql)
+            )
+            return False
+        else:
+            return True
+        finally:
+            self.close_connection(conn, cursor)
+
+    def add(self, sql, exception_callfunc=None):
+        """
+            单条 传入sql执行插入语句
+        :param sql: sql
+        :param exception_callfunc: 异常回调函数
+        :return: 添加行数
+        """
+        affect_count = None
+        conn, cursor = None, None
+
+        try:
+            conn, cursor = self.get_connection()
+            affect_count = cursor.execute(sql)
+            conn.commit()
+
+        except Exception as e:
+            log.error(
+                """
+                error:%s
+                sql:  %s
+            """
+                % (e, sql)
+            )
+            if exception_callfunc:
+                exception_callfunc(e)
+        finally:
+            self.close_connection(conn, cursor)
+
+        return affect_count
+
+    def add_smart(self, table, data: Dict, **kwargs):
+        """
+            单条 添加数据, 直接传递json格式的数据，不用拼sql
+        :param table: 表
+        :param data: 字典 {"xxx":"xxx"}
+        :param kwargs:
+        :return: 添加行数
+        """
+        sql = get_insert_sql(table, data, **kwargs)
+        return self.add(sql)
+
+    def add_batch(self, sql, datas: List[Dict]):
+        """
+            批量 添加数据
+            建议配合 get_insert_batch_sql() 生成sql
+            get_insert_batch_sql("user_copy1", [{"auth": 2, "id": "9", "email": "999"}]
+        :param sql:
+            insert ignore into `表` (字段1,字段2) values (%s, %s)
+            insert into `表` (`字段1`,`字段2`,`字段3`) values (%s, %s, %s)
+            这里有多少个字段，values后面就要有多少个%s
+        :param datas: 列表 [{}, {}, {}]
+        :return:
+        """
+        affect_count = None
+        conn, cursor = None, None
+        try:
+            conn, cursor = self.get_connection()
+            affect_count = cursor.executemany(sql, datas)
+            conn.commit()
+
+        except Exception as e:
+            log.error(
+                """
+                error:%s
+                sql:  %s
+                """
+                % (e, sql)
+            )
+        finally:
+            self.close_connection(conn, cursor)
+
+        return affect_count
+
+    def add_batch_smart(self, table, datas: List[Dict], **kwargs):
+        """
+            批量 直接传递list格式的数据，不用拼sql
+        :param table: 表名
+        :param datas: 列表 [{}, {}, {}]
+        :param kwargs:
+        :return: 添加行数
+        """
+        sql, datas = get_insert_batch_sql(table, datas, **kwargs)
+        return self.add_batch(sql, datas)
+
+    def update(self, sql):
+        """
+            更新
+        :param sql:
+        :return:
+        """
+        conn, cursor = None, None
+
+        try:
+            conn, cursor = self.get_connection()
+            cursor.execute(sql)
+            conn.commit()
+        except Exception as e:
+            log.error(
+                """
+                error:%s
+                sql:  %s
+            """
+                % (e, sql)
+            )
+            return False
+        else:
+            return True
+        finally:
+            self.close_connection(conn, cursor)
+
+    def update_smart(self, table, data: Dict, condition):
+        """
+            更新 无需拼sql
+        :param table: 表名
+        :param data: 数据 {"xxx":"xxx"}
+        :param condition: 更新条件 where后面的条件，如 condition='status=1'
+        :return:
+        """
+        sql = get_update_sql(table, data, condition)
+        return self.update(sql)
+
+    def delete(self, sql):
+        """
+            删除
+        :param sql:
+        :return:
+        """
+        conn, cursor = None, None
+        try:
+            conn, cursor = self.get_connection()
+            cursor.execute(sql)
+            conn.commit()
+        except Exception as e:
+            log.error(
+                """
+                error:%s
+                sql:  %s
+            """
+                % (e, sql)
+            )
+            return False
+        else:
+            return True
+        finally:
+            self.close_connection(conn, cursor)
+
+    @auto_retry
+    def find(self, sql, limit=0, to_json=False, conver_col=True):
+        """
+            查询
+            无数据： 返回None或[] 取决于limit
+            有数据： 如果limit=1 则返回 一条数据(字段1, 字段2) 其余返回[(字段1, 字段2),(字段1, 字段2)]
+        :param sql:
+        :param limit:
+        :param to_json: 是否将查询结果转为json
+        :param conver_col: 是否处理查询结果，如date类型转字符串，json字符串转成json。仅当to_json=True时生效
+        :return:
+        """
+        conn, cursor = self.get_connection()
+
+        cursor.execute(sql)
+
+        if limit == 1:
+            result = cursor.fetchone()  # 全部查出来，截取 不推荐使用
+        elif limit > 1:
+            result = cursor.fetchmany(limit)  # 全部查出来，截取 不推荐使用
+        else:
+            result = cursor.fetchall()
+
+        if result is None:
+            return result
+
+        if to_json:
+            columns = [i[0] for i in cursor.description]
+
+            # 处理数据
+            def convert(col):
+                if isinstance(col, (datetime.date, datetime.time)):
+                    return str(col)
+                elif isinstance(col, str) and (
+                        col.startswith("{") or col.startswith("[")
+                ):
+                    try:
+                        # col = self.unescape_string(col)
+                        return json.loads(col)
+                    except:
+                        return col
+                else:
+                    # col = self.unescape_string(col)
+                    return col
+
+            if limit == 1:
+                if conver_col:
+                    result = [convert(col) for col in result]
+                result = dict(zip(columns, result))
+            else:
+                if conver_col:
+                    result = [[convert(col) for col in row] for row in result]
+                result = [dict(zip(columns, r)) for r in result]
+
+        self.close_connection(conn, cursor)
+
+        return result
+
+
+if __name__ == '__main__':
+    pass
+    # mysql_db = MysqlDBPro(ip="127.0.0.1", port=3306, db="xtn_home", user_name="root", user_pass="xtn-kk")
+    # sql = """insert into `user_copy1` (`id`, `email`, `auth`) values (8, '888', 2)"""
+    # print(mysql_db.add(sql))
+    # print(mysql_db.add_smart("user_copy1", {"id": "9", "email": "999"}))
+    # sql = "insert ignore into `user_copy1` (`id`,`email`) values (%s, %s)"
+    # sql, datas = get_insert_batch_sql("user_copy1", [{"auth": 2, "id": "9", "email": "999"}])
+    # print(mysql_db.add_batch(sql, datas))
+
+    # print(mysql_db.add_batch_smart("user_copy1", [{"auth": 2, "id": "9", "email": "999"},
+    #                                               {"auth": 2, "id": "10", "email": "10"},
+    #                                               {"id": "11", "auth": 1, "email": "11"},
+    #                                               {"auth": 2, "id": "12", "email": "12"}]))
+
+    # 更新案例
+    # sql = "UPDATE user_copy1 SET status = '2', auth = 1 WHERE id = 2;"
+    # print(mysql_db.update(sql))
+
+    # 更新 无需拼接sql案例
+    # print(mysql_db.update_smart("user_copy1", {"email": "123", "status": 4}, "id=22"))
+
+    # 查询案例
+    # print(mysql_db.find("select * from user_copy1 where id=11",1,True))

xtn_tools_pro/tools.py

@@ -8,7 +8,8 @@
 # --------------------------------------------------------------------------------------------------
 # 2024/4/17    xiatn     V00.01.000    新建
 # --------------------------------------------------------------------------------------------------
-import hashlib, json, math
+import hashlib, json, math,re
+from pprint import pformat
 from urllib.parse import urlencode
 
 
@@ -128,6 +129,48 @@ def get_calculate_total_page(total, limit):
     total_pages = math.ceil(total / limit)
     return total_pages
 
+def list_to_strtuple(datas):
+    """
+        列表转字符串
+    :param datas: datas: [1, 2]
+    :return: (1, 2) 字符串类型
+    """
+    data_str = str(tuple(datas))
+    data_str = re.sub(",\)$", ")", data_str)
+    return data_str
+
+
+
+
+def dumps_json(data,indent=4,sort_keys=False):
+    """
+        将JSON数据格式化为可打印的字符串
+    :param data:
+    :param indent: 每一级嵌套都使用4个空格进行缩进
+    :param sort_keys: 是否排序
+    :return:
+    """
+    try:
+        if isinstance(data, str):
+            data = get_str_to_json(data)
+
+        data = json.dumps(
+            data,
+            ensure_ascii=False,
+            indent=indent,
+            skipkeys=True,
+            sort_keys=sort_keys,
+            default=str,
+        )
+
+    except Exception as e:
+        data = pformat(data)
+
+    return data
+
+
+def split_image(img):
+    pass
 
 if __name__ == '__main__':
     pass

xtn_tools_pro/tools_flie.py

@@ -38,4 +38,4 @@ def get_file_check_filename(file_name):
 if __name__ == '__main__':
     pass
     print(get_file_extension('file/2024-04-19/BOSCH GEX 125-1A/125-1AE砂磨机操作说明书:[1]_jingyan.txt'))
-    print(get_check_filename('file/2024-04-19/BOSCH GEX 125-1A/125-1AE砂磨机操作说明书:[1]_jingyan.txt'))
+    print(get_file_check_filename('file/2024-04-19/BOSCH GEX 125-1A/125-1AE砂磨机操作说明书:[1]_jingyan.txt'))

xtn_tools_pro/utils/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/5/12    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------

xtn_tools_pro/utils/log.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    日志
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/5/12     xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import os
+import sys
+import time
+import inspect
+import logging
+from xtn_tools_pro.tools_time import get_time_timestamp_to_datestr
+from logging.handlers import BaseRotatingHandler
+
+
+class RotatingFileHandler(BaseRotatingHandler):
+    def __init__(
+            self, filename, mode="a", max_bytes=0, backup_count=0, encoding=None, delay=0
+    ):
+        BaseRotatingHandler.__init__(self, filename, mode, encoding, delay)
+        self.max_bytes = max_bytes
+        self.backup_count = backup_count
+        self.placeholder = str(len(str(backup_count)))
+
+    def doRollover(self):
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+        if self.backup_count > 0:
+            for i in range(self.backup_count - 1, 0, -1):
+                sfn = ("%0" + self.placeholder + "d.") % i  # '%2d.'%i -> 02
+                sfn = sfn.join(self.baseFilename.split("."))
+                # sfn = "%d_%s" % (i, self.baseFilename)
+                # dfn = "%d_%s" % (i + 1, self.baseFilename)
+                dfn = ("%0" + self.placeholder + "d.") % (i + 1)
+                dfn = dfn.join(self.baseFilename.split("."))
+                if os.path.exists(sfn):
+                    # print "%s -> %s" % (sfn, dfn)
+                    if os.path.exists(dfn):
+                        os.remove(dfn)
+                    os.rename(sfn, dfn)
+            dfn = (("%0" + self.placeholder + "d.") % 1).join(
+                self.baseFilename.split(".")
+            )
+            if os.path.exists(dfn):
+                os.remove(dfn)
+            # Issue 18940: A file may not have been created if delay is True.
+            if os.path.exists(self.baseFilename):
+                os.rename(self.baseFilename, dfn)
+        if not self.delay:
+            self.stream = self._open()
+
+    def shouldRollover(self, record):
+
+        if self.stream is None:  # delay was set...
+            self.stream = self._open()
+        if self.max_bytes > 0:  # are we rolling over?
+            msg = "%s\n" % self.format(record)
+            self.stream.seek(0, 2)  # due to non-posix-compliant Windows feature
+            if self.stream.tell() + len(msg) >= self.max_bytes:
+                return 1
+        return 0
+
+
+class BoldFormatter(logging.Formatter):
+    def format(self, record):
+        result = super().format(record)
+        return "\033[1m" + result + "\033[0m"
+
+
+class Log:
+    def __init__(self, name, path=None, log_level='DEBUG',
+                 is_write_to_console=True,
+                 is_write_to_file=False,
+                 color=True,
+                 mode='a',
+                 max_bytes=0, backup_count=0, encoding=None):
+        """
+        :param name: log名
+        :param path: log文件存储路径 如 D://xxx.log
+        :param log_level: log等级 CRITICAL/ERROR/WARNING/INFO/DEBUG
+        :param is_write_to_console: 是否输出到控制台
+        :param is_write_to_file: 是否写入到文件 默认否
+        :param color: 是否有颜色
+        :param mode: 写文件模式
+        :param max_bytes: 每个日志文件的最大字节数
+        :param backup_count: 日志文件保留数量
+        :param encoding: 日志文件编码
+
+        """
+        # 创建logger对象
+        self.logger = logging.getLogger(name)
+        # 设置日志等级
+        self.logger.setLevel(log_level.upper())
+
+        # 创建日志格式化器
+        # formatter = logging.Formatter('[%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s') # 原
+        # formatter = logging.Formatter('\033[1m%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s\033[0m') #加粗
+        formatter = logging.Formatter(
+            '\033[1m[%(now_datestr)s] | %(levelname)-8s | [%(func_name)s] - %(message)s\033[0m')  # 加粗对齐
+
+        # formatter = BoldFormatter('[%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s') # 加粗
+
+        # 判断是否要输出到控制台
+        if is_write_to_console:
+            # 创建控制台处理器
+            console_handler = logging.StreamHandler(sys.stdout)
+            # 设置控制台处理器的格式化器
+            console_handler.setFormatter(formatter)
+            # 将控制台处理器添加到logger中
+            self.logger.addHandler(console_handler)
+
+        # 判断是否要写入文件
+        if is_write_to_file:
+            # 创建文件处理器
+            file_handler = RotatingFileHandler(path, mode=mode, max_bytes=max_bytes,
+                                               backup_count=backup_count, encoding=encoding)
+            # 设置文件处理器的格式化器
+            file_handler.setFormatter(formatter)
+            # 将文件处理器添加到logger中
+            self.logger.addHandler(file_handler)
+
+        # 判断是否要带颜色
+        if color:
+            try:
+                from colorlog import ColoredFormatter
+                # 创建带颜色的日志格式化器
+                # color_formatter = ColoredFormatter('%(log_color)s[%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s') # 原
+                # color_formatter = ColoredFormatter('\033[1m%(log_color)s[%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s\033[0m') # 加粗
+                # 创建颜色映射
+                log_colors = {
+                    'DEBUG': 'bold_blue',
+                    'INFO': 'bold_cyan',
+                    'WARNING': 'bold_yellow',
+                    'ERROR': 'bold_red',
+                    'CRITICAL': 'bold_red',
+                }
+                color_formatter = ColoredFormatter(
+                    '\033[1m%(log_color)s[%(now_datestr)s] | %(levelname)-8s | [%(func_name)s] - %(message)s\033[0m',
+                    log_colors=log_colors)  # 加粗对齐
+                # 设置控制台处理器的格式化器为带颜色的格式化器
+                console_handler.setFormatter(color_formatter)
+            except ImportError:
+                pass
+
+    def debug(self, message):
+        # 记录DEBUG级别的日志
+        self.logger.debug(message, extra=self._get_caller_name_extra())
+
+    def info(self, message):
+        # 记录INFO级别的日志
+        self.logger.info(message, extra=self._get_caller_name_extra())
+
+    def warning(self, message):
+        # 记录WARNING级别的日志
+        self.logger.warning(message, extra=self._get_caller_name_extra())
+
+    def error(self, message):
+        # 记录ERROR级别的日志
+        self.logger.error(message, extra=self._get_caller_name_extra())
+
+    def critical(self, message):
+        # 记录CRITICAL级别的日志
+        self.logger.critical(message, extra=self._get_caller_name_extra())
+
+    def _get_caller_name_extra(self):
+        """
+            获取调用日志函数的函数名称
+        """
+        # 获取当前栈帧
+        frame = inspect.currentframe()
+        # 获取调用者的栈帧
+        caller_frame = frame.f_back.f_back
+        # 从栈帧中获取代码对象
+        code_obj = caller_frame.f_code
+        # 获取调用者的名字
+        caller_name = code_obj.co_name
+        return {"func_name": caller_name,
+                "now_datestr": get_time_timestamp_to_datestr()}
+
+
+if __name__ == '__main__':
+    pass
+    logger = Log('mylogger', './xxx.log', log_level='DEBUG', is_write_to_console=True, is_write_to_file=True,
+                 color=True, mode='a', max_bytes=1024, backup_count=3)
+    logger.debug("debug message")
+    logger.info("info level message")
+    logger.warning("warning level message")
+    logger.critical("critical level message")

xtn_tools_pro/utils/retry.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    重试
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/5/12    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import time
+from xtn_tools_pro.utils.log import Log
+
+log = Log(name="retry", color=True)
+
+
+def retry(max_attempts=3, delay=0, exception_callfunc=None, *args_callfunc, **kwargs_callfunc):
+    """
+        重试
+    :param max_attempts: 最多重试次数
+    :param delay: 每次重试间隔时间
+    :param exception_callfunc: 失败的回调函数
+    :return:
+    """
+
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            attempts = 0
+            while attempts < max_attempts:
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    log.debug(f"重试第 {attempts + 1} 次,failed: {e}")
+                    if exception_callfunc:
+                        exception_callfunc(*args_callfunc, **kwargs_callfunc)
+                    attempts += 1
+                    time.sleep(delay)
+
+        return wrapper
+
+    return decorator
+
+
+if __name__ == '__main__':
+
+    def test1(*args, **kwargs):
+        print("test1", args, kwargs)
+
+
+    @retry(3, 5)
+    def test(a, b):
+        import random
+        if random.random() < 0.5:
+            raise ValueError("Random value too small")
+        print("Success!")
+
+    test(1, 1)

xtn_tools_pro/utils/sql.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    sql相关
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/5/12    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import datetime
+from xtn_tools_pro.tools import list_to_strtuple, dumps_json
+
+
+def format_sql_value(value):
+    if isinstance(value, str):
+        value = value.strip()
+
+    elif isinstance(value, (list, dict)):
+        value = dumps_json(value, indent=None)
+
+    elif isinstance(value, (datetime.date, datetime.time)):
+        value = str(value)
+
+    elif isinstance(value, bool):
+        value = int(value)
+
+    return value
+
+
+def get_insert_sql(table, data, auto_update=False, update_columns=(), insert_ignore=False):
+    """
+        生成 insert sql
+    :param table: 表
+    :param data: 表数据 json格式
+    :param auto_update: 使用的是replace into， 为完全覆盖已存在的数据
+    :param update_columns: 需要更新的列 默认全部，当指定值时，auto_update设置无效，当duplicate key冲突时更新指定的列
+    :param insert_ignore: 数据存在忽略，True则会忽略该插入操作，不会产生冲突错误
+    :return:
+    """
+    keys = ["`{}`".format(key) for key in data.keys()]
+    keys = list_to_strtuple(keys).replace("'", "")
+    values = [format_sql_value(value) for value in data.values()]
+    values = list_to_strtuple(values)
+    if update_columns:
+        if not isinstance(update_columns, (tuple, list)):
+            update_columns = [update_columns]
+        update_columns_ = ", ".join(
+            ["{key}=values({key})".format(key=key) for key in update_columns]
+        )
+        sql = (
+                "insert%s into `{table}` {keys} values {values} on duplicate key update %s"
+                % (" ignore" if insert_ignore else "", update_columns_)
+        )
+
+    elif auto_update:
+        sql = "replace into `{table}` {keys} values {values}"
+    else:
+        sql = "insert%s into `{table}` {keys} values {values}" % (
+            " ignore" if insert_ignore else ""
+        )
+
+    sql = sql.format(table=table, keys=keys, values=values).replace("None", "null")
+    return sql
+
+def get_insert_batch_sql(table, datas, auto_update=False, update_columns=(), update_columns_value=()):
+    """
+        生成 批量 insert sql
+    :param table: 表
+    :param datas: 表数据 [{...}]
+    :param auto_update: 使用的是replace into， 为完全覆盖已存在的数据
+    :param update_columns: 需要更新的列 默认全部，当指定值时，auto_update设置无效，当duplicate key冲突时更新指定的列
+    :param update_columns_value: 需要更新的列的值 默认为datas里边对应的值, 注意 如果值为字符串类型 需要主动加单引号， 如 update_columns_value=("'test'",)
+    :return:
+    """
+    if not datas:
+        return
+    keys = list(set([key for data in datas for key in data]))
+    values_placeholder = ["%s"] * len(keys)
+
+    values = []
+    for data in datas:
+        value = []
+        for key in keys:
+            current_data = data.get(key)
+            current_data = format_sql_value(current_data)
+
+            value.append(current_data)
+
+        values.append(value)
+
+    keys = ["`{}`".format(key) for key in keys]
+    keys = list_to_strtuple(keys).replace("'", "")
+
+    values_placeholder = list_to_strtuple(values_placeholder).replace("'", "")
+
+    if update_columns:
+        if not isinstance(update_columns, (tuple, list)):
+            update_columns = [update_columns]
+        if update_columns_value:
+            update_columns_ = ", ".join(
+                [
+                    "`{key}`={value}".format(key=key, value=value)
+                    for key, value in zip(update_columns, update_columns_value)
+                ]
+            )
+        else:
+            update_columns_ = ", ".join(
+                ["`{key}`=values(`{key}`)".format(key=key) for key in update_columns]
+            )
+        sql = "insert into `{table}` {keys} values {values_placeholder} on duplicate key update {update_columns}".format(
+            table=table,
+            keys=keys,
+            values_placeholder=values_placeholder,
+            update_columns=update_columns_,
+        )
+    elif auto_update:
+        sql = "replace into `{table}` {keys} values {values_placeholder}".format(
+            table=table, keys=keys, values_placeholder=values_placeholder
+        )
+    else:
+        sql = "insert ignore into `{table}` {keys} values {values_placeholder}".format(
+            table=table, keys=keys, values_placeholder=values_placeholder
+        )
+
+    return sql, values
+
+def get_update_sql(table, data, condition):
+    """
+        生成更新sql
+    :param table: 表
+    :param data: 表数据 json格式
+    :param condition: where 条件
+    :return:
+    """
+    key_values = []
+
+    for key, value in data.items():
+        value = format_sql_value(value)
+        if isinstance(value, str):
+            key_values.append("`{}`={}".format(key, repr(value)))
+        elif value is None:
+            key_values.append("`{}`={}".format(key, "null"))
+        else:
+            key_values.append("`{}`={}".format(key, value))
+
+    key_values = ", ".join(key_values)
+
+    sql = "update `{table}` set {key_values} where {condition}"
+    sql = sql.format(table=table, key_values=key_values, condition=condition)
+    return sql
+
+if __name__ == '__main__':
+    print(get_insert_sql("user_copy1", {"id": 5, "nickname": "1212", "email": "121212", "auth": 2},insert_ignore=True))
+    print(get_insert_batch_sql("user_copy1", [{"id": 5, "nickname": "555", "email": "555", "auth": 1},
+                                              {"id": 6, "nickname": "666", "email": "666", "auth": 2},
+                                              {"id": 7, "nickname": "777", "email": "777", "auth": 1}],
+                               ))
+    print(get_update_sql("user_copy1",{"email":"123","status":4},"id=2"))

{xtn_tools_pro-1.0.0.0.4.dist-info → xtn_tools_pro-1.0.0.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.0.4
+Version: 1.0.0.0.5
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@163.com
@@ -10,6 +10,9 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pymongo
 Requires-Dist: redis
+Requires-Dist: pymysql
+Requires-Dist: dbutils
+Requires-Dist: colorlog
 Requires-Dist: requests
 
 xtnkk-tools

xtn_tools_pro-1.0.0.0.5.dist-info/RECORD

@@ -0,0 +1,19 @@
+xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
+xtn_tools_pro/tools.py,sha256=xOqTL7jrJvmsPZBiUizKe7yWkmqvEbt-WGylGswNcQ8,4457
+xtn_tools_pro/tools_flie.py,sha256=-FID31G8AnV2u2djRO3Ae3Ei9ld9bSBWgzwqhYzZ5JQ,1517
+xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
+xtn_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
+xtn_tools_pro/db/MysqlDB.py,sha256=SBJrcjbZdxmtTKPGwl57NthPhs4uX8J3P6o_rK01O4k,13373
+xtn_tools_pro/db/RedisDB.py,sha256=ep32Yj8AAkUHRshSBhKsdl06UwO7Z-gQJLaezspVRKw,6053
+xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
+xtn_tools_pro/proxy/XiaoXiangProxy.py,sha256=xrEBJZ6Cjuh6IBZVB17oXHwByaeKcWVUSSWSg17tujE,9842
+xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmcyo,418
+xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
+xtn_tools_pro/utils/log.py,sha256=bSuPsGKVBbGLT5hmIcfRi9TeuJApALcPa1qGMrCHk24,8146
+xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
+xtn_tools_pro/utils/sql.py,sha256=YSDx2BxkQk-QA6RGBiImR9bbT4WyKDiCYU8Q4ZdaMG0,6255
+xtn_tools_pro-1.0.0.0.5.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.0.5.dist-info/METADATA,sha256=JAHmDIpLXqWb8Y5SoEso1ss7gh09BfnMEhmUIBc_D6c,431
+xtn_tools_pro-1.0.0.0.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.0.5.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.0.5.dist-info/RECORD,,

xtn_tools_pro-1.0.0.0.4.dist-info/RECORD

@@ -1,14 +0,0 @@
-xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
-xtn_tools_pro/tools.py,sha256=-EeI_T7jEccapvZ6Jz25Ta0SIunqqcvDSWP2qn5b4aU,3502
-xtn_tools_pro/tools_flie.py,sha256=B_P3J_R-nRLt_IFutnOVrBRGf6_SZ_cXoIoeaT9B7tk,1512
-xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
-xtn_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
-xtn_tools_pro/db/RedisDB.py,sha256=ep32Yj8AAkUHRshSBhKsdl06UwO7Z-gQJLaezspVRKw,6053
-xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
-xtn_tools_pro/proxy/XiaoXiangProxy.py,sha256=xrEBJZ6Cjuh6IBZVB17oXHwByaeKcWVUSSWSg17tujE,9842
-xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmcyo,418
-xtn_tools_pro-1.0.0.0.4.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.0.4.dist-info/METADATA,sha256=nLLM4xF-cM4V6dqxghgyDjq6B2BAqHm0jI07PIuflQU,358
-xtn_tools_pro-1.0.0.0.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.0.4.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.0.4.dist-info/RECORD,,