mdbq 2.9.0__py3-none-any.whl → 2.9.2__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -41,6 +41,63 @@ warnings.filterwarnings('ignore')
 """
 
 
+def is_valid_date(date_string):
+    """ 判断是否是日期格式, 且允许没有前导零, 且允许带时间 """
+    date_pattern = r"^(\d{4})-(0?[1-9]|1[0-2])-(0?[1-9]|[12]\d|3[01])$"
+    match = re.match(date_pattern, str(date_string))  # 判断纯日期格式：2024-11-09
+    if match is None:
+        date_pattern = r".*\d+:\d+:\d+$"
+        match = re.match(date_pattern, date_string)  # 判断日期+时间：2024-11-09 00:36:45
+        if match is not None:
+            return 2
+    else:
+        return 1
+
+
+def is_integer(int_str):
+    """ 判断是否整数, 允许包含千分位分隔符, 允许科学计数法 """
+    # 如果是科学计数法
+    match = re.findall(r'^[-+]?(\d+)\.(\d+)[eE][-+]?(\d+)$', str(int_str))
+    if match:
+        if len(match[0]) == 3:
+            if int(match[0][0]) == 0:  # 0 开头
+                if int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            else:  # 不是 0 开头
+                if len(match[0][0]) + int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            if int(match[0][2]) >= len(match[0][1]):
+                return True
+            else:
+                return False
+    # 如果是普通数字, 且允许千分符
+    __pattern = r'^[-+]?\d{1,3}(,\d{3}){0,3}$|^[-+]?\d{1,9}$'
+    return re.match(__pattern, str(int_str)) is not None
+
+
+def count_decimal_places(num_str):
+    """ 计算小数位数, 允许科学计数法 """
+    match = re.match(r'^[-+]?\d+(\.\d+)?([eE][-+]?\d+)?$', str(num_str))
+    if match:
+        # 如果是科学计数法
+        match = re.findall(r'(\d+)\.(\d+)[eE][-+]?(\d+)$', str(num_str))
+        if match:
+            if len(match[0]) == 3:
+                if int(match[0][2]) < len(match[0][1]):
+                    # count_int 清除整数部分开头的 0 并计算整数位数
+                    count_int = len(re.sub('^0+', '', str(match[0][0]))) + int(match[0][2])
+                    # 计算小数位数
+                    count_float = len(match[0][1]) - int(match[0][2])
+                    return count_int, count_float
+        # 如果是普通小数
+        match = re.findall(r'(\d+)\.(\d+)$', str(num_str))
+        if match:
+            count_int = len(re.sub('^0+', '', str(match[0][0])))
+            count_float = len(match[0][1])
+            return count_int, count_float  # 计算小数位数
+    return 0, 0
+
+
 class MysqlUpload:
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
         self.username = username
@@ -68,6 +125,127 @@ class MysqlUpload:
 
         return wrapper
 
+    def dict_to_mysql(self, db_name, table_name, dict_data, main_key=None, unique_main_key=None, index_length=100):
+        """
+        插入字典数据
+        dict_data： 字典
+        main_key： 指定索引列
+        unique_main_key： 指定唯一索引列
+        index_length: 索引长度
+        """
+
+        if not main_key:
+            main_key = []
+        if not unique_main_key:
+            unique_main_key = []
+        connection = pymysql.connect(**self.config)  # 连接数据库
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            database_exists = cursor.fetchone()
+            if not database_exists:
+                # 如果数据库不存在，则新建
+                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
+                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
+                if '192.168.1.100' in str(self.host):
+                    sql = f"CREATE DATABASE `{db_name}`"
+                else:
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                cursor.execute(sql)
+                connection.commit()
+                print(f"创建Database: {db_name}")
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            # 1. 查询表, 不存在则创建一个空表
+            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
+            cursor.execute(sql, (table_name))
+            if not cursor.fetchone():
+                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
+                cursor.execute(sql)
+                print(f'创建 mysql 表: {table_name}')
+
+            # 根据 dict_data 的值添加指定的数据类型
+            dtypes = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'mediumtext',...}
+            # 检查列
+            sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
+            cursor.execute(sql, (db_name, table_name))
+            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]  # 已存在的所有列
+            col_not_exist = [col for col in dict_data.keys() if col not in col_exist]  # 不存在的列
+            # 不存在则新建列
+            if col_not_exist:  # 数据表中不存在的列
+                for col in col_not_exist:
+                    #  创建列，需转义
+                    sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    cursor.execute(sql)
+                    print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                    if col in main_key or col in unique_main_key:
+                        sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
+                        cursor.execute(sql, (col))
+                        result = cursor.fetchone()  # 检查索引是否存在
+                        if not result:
+                            if col in main_key:
+                                cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`({index_length}))")
+                                print(f"创建索引列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+                            elif col in unique_main_key:
+                                cursor.execute(f"CREATE UNIQUE INDEX index_name ON `{table_name}`(`{col}`({index_length}))")
+                                print(f"创建唯一索引: {col}({dtypes[col]})")  # 添加列并指定数据类型
+            connection.commit()  # 提交事务
+
+            keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
+            values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
+            del dict_data['客户id']
+            update_datas = ', '.join([f'{k} = VALUES({k})' for k, v in dict_data.items()])
+            # print(update_datas)
+            sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
+            # print(sql)
+            cursor.execute(sql)
+            connection.commit()  # 提交事务
+        connection.close()
+
+
+    def cover_dict_dtypes(self, dict_data):
+        if not dict_data:
+            print(f'mysql.py -> MysqlUpload -> cover_dict_dtypes -> 传入的字典不能为空')
+            return
+        __res_dict = {}
+        for k, v in dict_data.items():
+            result1 = re.findall(r'商品编码|[商品宝贝]_?id|s[kp]u_?id|货号|款号|文件大小', str(k), re.IGNORECASE)
+            result2 = re.findall(r'占比|投产|产出|同比|环比|roi$', str(k), re.IGNORECASE)
+            date_type = is_valid_date(str(v))  # 判断日期时间
+            int_num = is_integer(str(v))  # 判断整数
+            count_int, count_float = count_decimal_places(str(v))  # 判断小数，返回小数位数
+            if result1:  # 京东sku/spu商品信息
+                __res_dict.update({k: 'mediumtext'})
+            elif result2:
+                __res_dict.update({k: 'decimal(10,2)'})
+            elif str(v) == '' or str(v).lower() == 'nan' or str(v).lower() == 'null':
+                v = 0
+                dict_data.update({k: v})
+                __res_dict.update({k: 'mediumtext'})
+            elif date_type == 1:
+                __res_dict.update({k: 'DATE'})
+            elif date_type == 2:
+                __res_dict.update({k: 'DATETIME'})
+            elif int_num:
+                __res_dict.update({k: 'INT'})
+            elif count_float > 0:
+                if count_int + count_float > 10:
+                    __res_dict.update({k: 'mediumtext'})
+                elif count_float >= 6:
+                    __res_dict.update({k: 'decimal(12,6)'})
+                elif count_float >= 4:
+                    __res_dict.update({k: 'decimal(10,4)'})
+                else:
+                    __res_dict.update({k: 'decimal(10,2)'})
+            else:
+                __res_dict.update({k: 'mediumtext'})
+        return __res_dict
+
+
+
     @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'xigua_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """

{mdbq-2.9.0.dist-info → mdbq-2.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.9.0
+Version: 2.9.2
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.9.0.dist-info → mdbq-2.9.2.dist-info}/RECORD

@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=apcj0WDdbrHr7UzO2kjcesDxDUlWxG4KcIpI1mBuwMk,46152
+mdbq/mysql/mysql.py,sha256=jq2_lQFTnQR6N6QdSqxKqjennazITh5TdTg6j7oerYE,55006
 mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.9.0.dist-info/METADATA,sha256=s0ZT23hnWNpOZ5YBPTKigR8XftMlBOTCzH_7Lfuamig,243
-mdbq-2.9.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.9.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.9.0.dist-info/RECORD,,
+mdbq-2.9.2.dist-info/METADATA,sha256=3_u-jNgpFq2qX6uv2ufoYgPirGomwoz9IZkYmRp1BIc,243
+mdbq-2.9.2.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.9.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.2.dist-info/RECORD,,

{mdbq-2.9.0.dist-info → mdbq-2.9.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any