mdbq 3.8.13__py3-none-any.whl → 3.8.14__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -141,135 +141,17 @@ class MysqlUpload:
         return __res_dict, new_dict_data
 
     @try_except
-    def doc_to_sql(self, db_name, table_name, dict_data, set_typ={}, remove_by_key=None, allow_not_null=False, filename=None, reset_id=False):
-        """
-        db_name:
-        table_name:
-        remove_by_key: 设置时先删除数据再插入，不设置则直接添加
-        dict_data:
-        set_typ:
-        allow_not_null:
-        filename:
-        reset_id:
-        """
-        if not self.config:
-            return
-        if '数据主体' not in dict_data.keys():
-            logger.info(f'dict_data 中"数据主体"键不能为空')
-            return
-        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
-        if not connection:
-            return
-        with connection.cursor() as cursor:
-            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
-            database_exists = cursor.fetchone()
-            if not database_exists:
-                # 如果数据库不存在，则新建
-                sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
-                cursor.execute(sql)
-                connection.commit()
-                logger.info(f"创建Database: {db_name}")
-
-        self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
-        if not connection:
-            return
-        with connection.cursor() as cursor:
-            # 1. 查询表, 不存在则创建一个空表
-            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
-            cursor.execute(sql, (table_name))
-            if not cursor.fetchone():
-                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
-                cursor.execute(sql)
-                logger.info(f'创建 mysql 表: {table_name}')
-
-            new_dict = {}
-            [new_dict.update({k: v}) for k, v in dict_data.items() if k != '数据主体']
-            # 清理列名中的非法字符
-            dtypes, new_dict = self.cover_doc_dtypes(new_dict)
-            if set_typ:
-                # 更新自定义的列数据类型
-                for k, v in dtypes.items():
-                    # 确保传进来的 set_typ 键存在于实际的 df 列才 update
-                    [dtypes.update({k: inside_v}) for inside_k, inside_v in set_typ.items() if k == inside_k]
-
-            # 检查列
-            sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
-            cursor.execute(sql, (db_name, table_name))
-            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]  # 已存在的所有列
-
-            col_not_exist = [col for col in set_typ.keys() if col not in col_exist]  # 不存在的列
-            # 不存在则新建列
-            if col_not_exist:  # 数据表中不存在的列
-                for col in col_not_exist:
-                    #  创建列，需转义
-                    if allow_not_null:
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {set_typ[col]};"
-                    else:
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {set_typ[col]} NOT NULL;"
-                    cursor.execute(sql)
-                    logger.info(f"添加列: {col}({set_typ[col]})")  # 添加列并指定数据类型
-
-                    if col == '日期':
-                        sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
-                        logger.info(f"设置为索引: {col}({set_typ[col]})")
-                        cursor.execute(sql)
-            connection.commit()  # 提交事务
-
-            if remove_by_key:
-                # 删除数据
-                se_key = ', '.join(remove_by_key)
-                condition = []
-                for up_col in remove_by_key:
-                    condition += [f'`{up_col}` = "{dict_data[up_col]}"']
-                condition = ' AND '.join(condition)
-                sql = f"SELECT {se_key} FROM `{table_name}` WHERE {condition}"
-                cursor.execute(sql)
-                result = cursor.fetchall()
-                if result:
-                    sql = f'DELETE FROM `{table_name}` WHERE {condition};'
-                    cursor.execute(sql)
-
-            # 插入数据到数据库
-            # 有数据格式错误问题，所以分开处理，将数据主体移到最后面用占位符
-            logger.info(f'正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name} -> {filename}')
-            if new_dict:
-                cols = ', '.join(f"`{item}`" for item in new_dict.keys())  # 列名需要转义
-                values = ', '.join([f'"{item}"' for item in new_dict.values()])  # 值要加引号
-                cols = ', '.join([cols, '数据主体'])
-                binary_data = dict_data['数据主体']
-                sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values}, %s)"
-                cursor.execute(sql, binary_data)
-            else:
-                sql = f"""INSERT INTO `{table_name}` (数据主体) VALUES (%s);"""
-                cursor.execute(sql, dict_data['数据主体'])
-
-            if reset_id:
-                pass
-            connection.commit()
-
-    @try_except
-    def insert_many_dict(self, db_name, table_name, dict_data_list, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_typ=None, allow_not_null=False, cut_data=None):
+    def insert_many_dict(self, db_name, table_name, dict_data_list, icm_update=None, index_length=100, set_typ=None, allow_not_null=False, cut_data=None):
         """
         插入字典数据
         dict_data： 字典
-        main_key： 指定索引列, 通常用日期列，默认会设置日期为索引
-        unique_main_key： 指定唯一索引列
         index_length: 索引长度
-        icm_update: 增量更正，指定后 main_key 只用于检查/创建列，不能更新数据
+        icm_update: 增量更正
         set_typ: {}
         allow_not_null: 创建允许插入空值的列，正常情况下不允许空值
         """
         if not self.config:
             return
-        if icm_update:
-            if main_key or unique_main_key:
-                logger.info(f'icm_update/unique_main_key/unique_main_key 参数不能同时设定')
-                return
-        if not main_key:
-            main_key = []
-        if not unique_main_key:
-            unique_main_key = []
 
         if not dict_data_list:
             logger.info(f'dict_data_list 不能为空 ')
@@ -289,7 +171,6 @@ class MysqlUpload:
                 except Exception as e:
                     logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
 
-        # connection = pymysql.connect(**self.config)  # 连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
         if not connection:
             return
@@ -298,14 +179,12 @@ class MysqlUpload:
             database_exists = cursor.fetchone()
             if not database_exists:
                 # 如果数据库不存在，则新建
-
                 sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
                 cursor.execute(sql)
                 connection.commit()
                 logger.info(f"创建Database: {db_name}")
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        # connection = pymysql.connect(**self.config)  # 重新连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
         if not connection:
             return
@@ -339,47 +218,38 @@ class MysqlUpload:
                         sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
                     else:
                         sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
-                    # sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
-                    # logger.info(sql)
+
                     cursor.execute(sql)
                     logger.info(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
-                    if col in main_key or col == '日期':
+                    if col == '日期':
                         sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
                         logger.info(f"设置为索引: {col}({dtypes[col]})")
                         cursor.execute(sql)
-                    if col in unique_main_key:
-                        if dtypes[col] == 'mediumtext':
-                            sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`({index_length}))"
-                        else:
-                            sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`)"
-                        cursor.execute(sql)
+
             connection.commit()  # 提交事务
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
             # 处理插入的数据
             for dict_data in dict_data_list:
-                # logger.info(dict_data)
                 dtypes, dict_data = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'varchar(100)',...}
                 if icm_update:
                     """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列组合是数据表中唯一，值不会发生变化且不会重复，否则可能产生覆盖 """
                     sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, {table_name}))
+                    cursor.execute(sql, (db_name, table_name))
                     columns = cursor.fetchall()
                     cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
                     update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
 
-                    # unique_keys 示例: `日期`, `余额`
-                    unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
-                    condition = []
-                    for up_col in icm_update:
-                        condition += [f'`{up_col}` = "{dict_data[up_col]}"']
-                    condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
-                    sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
-                    # logger.info(sql)
-                    # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                    cursor.execute(sql)
-                    results = cursor.fetchall()  # results 是数据库取出的数据
+                    # 构造查询条件（参数化）
+                    condition = ' AND '.join([f'`{up_col}` = %s' for up_col in icm_update])
+                    condition_values = [dict_data[up_col] for up_col in icm_update]
+
+                    # 执行查询（参数化）
+                    sql = f"SELECT {','.join([f'`{col}`' for col in update_col])} FROM `{table_name}` WHERE {condition}"
+                    cursor.execute(sql, condition_values)
+                    results = cursor.fetchall()
+
                     if results:  # 有数据返回，再进行增量检查
                         for result in results:  # results 是数据库数据, dict_data 是传进来的数据
                             change_col = []  # 发生变化的列名
@@ -395,86 +265,61 @@ class MysqlUpload:
                                     mysql_value = re.sub(r'0+$', '', mysql_value)
                                     mysql_value = re.sub(r'\.$', '', mysql_value)
                                 if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
-                                    # logger.info(f'{dict_data['日期']}{dict_data['商品id']}{col} 列的值有变化，{str(dict_data[col])}  !=  {str(result[col])}')
                                     change_values += [f"`{col}` = \"{str(dict_data[col])}\""]
                                     change_col.append(col)
                             not_change_col = [item for item in update_col if item not in change_col]
-                            # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
-                            if change_values:  # change_values 有数据返回，表示值需要更新
+
+                            # 构造更新语句（参数化）
+                            if change_values:
+                                set_clause = ', '.join([f'`{col}` = %s' for col in change_col])
+                                update_values = [dict_data[col] for col in change_col]
+                                # 添加未变化列的查询条件
                                 if not_change_col:
-                                    not_change_values = [f'`{col}` = "{str(dict_data[col])}"' for col in not_change_col]
-                                    not_change_values = ' AND '.join(
-                                        not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
-                                    # logger.info(change_values, not_change_values)
-                                    condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
-                                change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
-                                sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
-                                # logger.info(sql)
-                                cursor.execute(sql)
+                                    not_change_condition = ' AND '.join([f'`{col}` = %s' for col in not_change_col])
+                                    condition += f' AND {not_change_condition}'
+                                    condition_values += [dict_data[col] for col in not_change_col]
+                                # 执行更新
+                                sql = f"UPDATE `{table_name}` SET {set_clause} WHERE {condition}"
+                                cursor.execute(sql, update_values + condition_values)
                     else:  # 没有数据返回，则直接插入数据
-                        # cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
-                        # # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                        # values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
-                        # sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                        # cursor.execute(sql)
-
-                        # 清理和验证列名
-                        safe_columns = [f"`{escape_string(str(col))}`" for col in dict_data.keys()]
-                        cols = ", ".join(safe_columns)
-                        # 使用参数化查询
-                        placeholders = ", ".join(["%s"] * len(dict_data))
+                        # 参数化插入
+                        cols = ', '.join([f'`{k}`' for k in dict_data.keys()])
+                        placeholders = ', '.join(['%s'] * len(dict_data))
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({placeholders})"
-                        # 转义值并作为参数传递
-                        escaped_values = [escape_string(str(v)) if isinstance(v, str) else v for v in
-                                          dict_data.values()]
-                        cursor.execute(sql, tuple(escaped_values))
-
+                        cursor.execute(sql, tuple(dict_data.values()))
                     connection.commit()  # 提交数据库
                     continue
 
-                # 构建 keys
-                keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
-                # 构建 values
-                values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
-                # 构建其他键值，重复时要更新的其他键
-                if main_key:
-                    for col in main_key:
-                        del dict_data[col]
-                if unique_main_key:
-                    for col in unique_main_key:
-                        del dict_data[col]
-                # 涉及列名务必使用反引号
-                update_datas = ', '.join([f'`{k}` = VALUES(`{k}`)' for k, v in dict_data.items()])
-
-                # 构建 sql
-                sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
-                # logger.info(sql)
-                cursor.execute(sql)
-                connection.commit()  # 提交数据库
+                # 标准插入逻辑（参数化修改）
+                # 构造更新列（排除主键）
+                update_cols = [k for k in dict_data.keys()]
+                # 构建SQL
+                cols = ', '.join([f'`{k}`' for k in dict_data.keys()])
+                placeholders = ', '.join(['%s'] * len(dict_data))
+                update_clause = ', '.join([f'`{k}` = VALUES(`{k}`)' for k in update_cols]) or 'id=id'
+
+                sql = f"""INSERT INTO `{table_name}` ({cols}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_clause}"""
+                # 执行参数化查询
+                try:
+                    cursor.execute(sql, tuple(dict_data.values()))
+                    connection.commit()
+                except pymysql.Error as e:
+                    logger.error(f"插入失败: {e}\nSQL: {cursor.mogrify(sql, tuple(dict_data.values()))}")
+                    connection.rollback()
         connection.close()
 
     @try_except
-    def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_typ=None, allow_not_null=False, cut_data=None):
+    def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, index_length=100, set_typ=None, allow_not_null=False, cut_data=None):
         """
         插入字典数据
         dict_data： 字典
-        main_key： 指定索引列, 通常用日期列，默认会设置日期为索引
-        unique_main_key： 指定唯一索引列
         index_length: 索引长度
-        icm_update: 增量更正，指定后 main_key 只用于检查/创建列，不能更新数据
+        icm_update: 增量更新
         set_typ: {}
         allow_not_null: 创建允许插入空值的列，正常情况下不允许空值
         """
         if not self.config:
             return
-        if icm_update:
-            if main_key or unique_main_key:
-                logger.info(f'icm_update/unique_main_key/unique_main_key 参数不能同时设定')
-                return
-        if not main_key:
-            main_key = []
-        if not unique_main_key:
-            unique_main_key = []
 
         if cut_data:
             if '日期' in dict_data.keys():
@@ -490,7 +335,6 @@ class MysqlUpload:
                 except Exception as e:
                     logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
 
-        # connection = pymysql.connect(**self.config)  # 连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
         if not connection:
             return
@@ -505,7 +349,6 @@ class MysqlUpload:
                 logger.info(f"创建Database: {db_name}")
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        # connection = pymysql.connect(**self.config)  # 重新连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
         if not connection:
             return
@@ -539,44 +382,34 @@ class MysqlUpload:
                         sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
                     else:
                         sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
-                    # sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
-                    # logger.info(sql)
                     cursor.execute(sql)
                     logger.info(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
-                    if col in main_key or col == '日期':
+                    if col == '日期':
                         sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
                         logger.info(f"设置为索引: {col}({dtypes[col]})")
                         cursor.execute(sql)
-                    if col in unique_main_key:
-                        if dtypes[col] == 'mediumtext':
-                            sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`({index_length}))"
-                        else:
-                            sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`)"
-                        cursor.execute(sql)
             connection.commit()  # 提交事务
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
             """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
             # 处理插入的数据
             if icm_update:
                 """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列组合是数据表中唯一，值不会发生变化且不会重复，否则可能产生覆盖 """
-                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                cursor.execute(sql, (db_name, {table_name}))
-                columns = cursor.fetchall()
-                cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+                sql = """SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s"""
+                cursor.execute(sql, (db_name, table_name))
+                cols_exist = [col['COLUMN_NAME'] for col in cursor.fetchall()] # 数据表的所有列, 返回 list
                 update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
 
-                # unique_keys 示例: `日期`, `余额`
-                unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
-                condition = []
+                # 参数化构建查询条件
+                condition_params = []
+                condition_parts = []
                 for up_col in icm_update:
-                    condition += [f'`{up_col}` = "{dict_data[up_col]}"']
-                condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
-                sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
-                # logger.info(sql)
-                # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                cursor.execute(sql)
-                results = cursor.fetchall()  # results 是数据库取出的数据
+                    condition_parts.append(f"`{up_col}` = %s")
+                    condition_params.append(dict_data[up_col])
+
+                sql = f"""SELECT `{','.join(update_col)}` FROM `{table_name}` WHERE {' AND '.join(condition_parts)}"""
+                cursor.execute(sql, condition_params)
+                results = cursor.fetchall()
                 if results:  # 有数据返回，再进行增量检查
                     for result in results:  # results 是数据库数据, dict_data 是传进来的数据
                         change_col = []  # 发生变化的列名
@@ -592,62 +425,35 @@ class MysqlUpload:
                                 mysql_value = re.sub(r'0+$', '', mysql_value)
                                 mysql_value = re.sub(r'\.$', '', mysql_value)
                             if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
-                                # logger.info(f'{dict_data['日期']}{dict_data['商品id']}{col} 列的值有变化，{str(dict_data[col])}  !=  {str(result[col])}')
                                 change_values += [f"`{col}` = \"{str(dict_data[col])}\""]
                                 change_col.append(col)
-                        not_change_col = [item for item in update_col if item not in change_col]
                         # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
                         if change_values:  # change_values 有数据返回，表示值需要更新
-                            if not_change_col:
-                                not_change_values = [f'`{col}` = "{str(dict_data[col])}"' for col in not_change_col]
-                                not_change_values = ' AND '.join(
-                                    not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
-                                # logger.info(change_values, not_change_values)
-                                condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
-                            change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
-                            sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
-                            # logger.info(sql)
-                            cursor.execute(sql)
-                else:  # 没有数据返回，则直接插入数据
-                    # cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
-                    # # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    # values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
-                    # sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                    # cursor.execute(sql)
-
-                    # 构建安全的INSERT语句
-                    safe_keys = [f"`{escape_string(str(k))}`" for k in dict_data.keys()]
-                    keys_data = ", ".join(safe_keys)
-                    placeholders = ", ".join(["%s"] * len(dict_data))
-
-                    # 使用参数化查询
-                    sql = f"INSERT INTO `{table_name}` ({keys_data}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_datas}"
-                    escaped_values = [escape_string(str(v)) if isinstance(v, str) else v for v in dict_data.values()]
-                    cursor.execute(sql, tuple(escaped_values))
+                            set_params = [dict_data[col] for col in change_col]
+                            full_params = set_params + condition_params  # 正确顺序
 
+                            sql = f"""UPDATE `{table_name}` 
+                                    SET {','.join(set_parts)} 
+                                    WHERE {' AND '.join(condition_parts)}"""
+                            cursor.execute(sql, full_params)
+                else:  # 没有数据返回，则直接插入数据
+                    # 参数化插入语句
+                    keys = [f"`{k}`" for k in dict_data.keys()]
+                    placeholders = ','.join(['%s'] * len(dict_data))
+                    update_clause = ','.join([f"`{k}`=VALUES(`{k}`)" for k in dict_data.keys()])
+                    sql = f"""INSERT INTO `{table_name}` ({','.join(keys)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_clause}"""
+                    cursor.execute(sql, tuple(dict_data.values()))
                 connection.commit()  # 提交数据库
                 connection.close()
                 return
 
-            # 构建 keys
-            keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
-            # 构建 values
-            values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
-            # 构建其他键值，重复时要更新的其他键
-            if main_key:
-                for col in main_key:
-                    del dict_data[col]
-            if unique_main_key:
-                for col in unique_main_key:
-                    del dict_data[col]
-            # 涉及列名务必使用反引号
-            update_datas = ', '.join([f'`{k}` = VALUES(`{k}`)' for k, v in dict_data.items()])
-
-            # 构建 sql
-            sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
-            # logger.info(sql)
-            cursor.execute(sql)
-            connection.commit()  # 提交数据库
+            # 常规插入处理（参数化）
+            keys = [f"`{k}`" for k in dict_data.keys()]
+            placeholders = ','.join(['%s'] * len(dict_data))
+            update_clause = ','.join([f"`{k}`=VALUES(`{k}`)" for k in dict_data.keys()])
+            sql = f"""INSERT INTO `{table_name}` ({','.join(keys)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE {update_clause}"""
+            cursor.execute(sql, tuple(dict_data.values()))
+            connection.commit()
         connection.close()
 
     def cover_dict_dtypes(self, dict_data):
@@ -749,17 +555,15 @@ class MysqlUpload:
         return __res_dict, df
 
     @try_except
-    def df_to_mysql(self, df, db_name, table_name, set_typ=None, icm_update=[], move_insert=False, df_sql=False, drop_duplicates=False,
-                    filename=None, count=None, reset_id=False, allow_not_null=False, cut_data=None):
+    def df_to_mysql(self, df, db_name, table_name, set_typ=None, icm_update=[], move_insert=False, df_sql=False,
+                    filename=None, count=None, allow_not_null=False, cut_data=None):
         """
         db_name: 数据库名
         table_name: 表名
-        move_insert: 根据df 的日期，先移除数据库数据，再插入, df_sql, drop_duplicates, icm_update 都要设置为 False
+        move_insert: 根据df 的日期，先移除数据库数据，再插入, df_sql, icm_update 都要设置为 False
         原则上只限于聚合数据使用，原始数据插入时不要设置
-
         df_sql: 这是一个临时参数, 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重，初创表大量上传数据的时候使用
-        drop_duplicates: 值为 True 时检查重复数据再插入，反之直接上传，数据量大时会比较慢
-        icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用，设置此参数时需将 drop_duplicates 改为 False
+        icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用
                 使用增量更新: 必须确保 icm_update 传进来的列必须是数据表中唯一主键，值不会发生变化，不会重复，否则可能产生错乱覆盖情况
         filename: 用来追踪处理进度，传这个参数是方便定位产生错误的文件
         allow_not_null: 创建允许插入空值的列，正常情况下不允许空值
@@ -767,12 +571,12 @@ class MysqlUpload:
         if not self.config:
             return
         if icm_update:
-            if move_insert or df_sql or drop_duplicates:
-                logger.info(f'icm_update/move_insert/df_sql/drop_duplicates 参数不能同时设定')
+            if move_insert or df_sql:
+                logger.info(f'icm_update/move_insert/df_sql 参数不能同时设定')
                 return
         if move_insert:
-            if icm_update or df_sql or drop_duplicates:
-                logger.info(f'icm_update/move_insert/df_sql/drop_duplicates 参数不能同时设定')
+            if icm_update or df_sql:
+                logger.info(f'icm_update/move_insert/df_sql 参数不能同时设定')
                 return
 
         self.filename = filename
@@ -811,12 +615,11 @@ class MysqlUpload:
                 # 确保传进来的 set_typ 键存在于实际的 df 列才 update
                 [dtypes.update({k: inside_v}) for inside_k, inside_v in set_typ.items() if k == inside_k]
 
-        # connection = pymysql.connect(**self.config)  # 连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
         if not connection:
             return
         with connection.cursor() as cursor:
-            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            cursor.execute("SHOW DATABASES LIKE %s", (db_name,))  # 检查数据库是否存在
             database_exists = cursor.fetchone()
             if not database_exists:
                 # 如果数据库不存在，则新建
@@ -826,7 +629,6 @@ class MysqlUpload:
                 logger.info(f"创建Database: {db_name}")
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
-        # connection = pymysql.connect(**self.config)  # 重新连接数据库
         connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
         if not connection:
             return
@@ -835,8 +637,8 @@ class MysqlUpload:
             sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
             cursor.execute(sql, (table_name))
             if not cursor.fetchone():
-                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
-                cursor.execute(sql)
+                create_table_sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY)"
+                cursor.execute(create_table_sql)
                 logger.info(f'创建 mysql 表: {table_name}')
 
             #  有特殊字符不需转义
@@ -850,11 +652,10 @@ class MysqlUpload:
             if col_not_exist:  # 数据表中不存在的列
                 for col in col_not_exist:
                     #  创建列，需转义
-                    if allow_not_null:
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
-                    else:
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
-                    cursor.execute(sql)
+                    ialter_sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]}"
+                    if not allow_not_null:
+                        alter_sql += " NOT NULL"
+                    cursor.execute(alter_sql)
                     logger.info(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
                     # 创建索引
@@ -870,17 +671,14 @@ class MysqlUpload:
                 logger.info(f'正在更新: mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count}, {self.filename}')
                 engine = create_engine(
                     f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
-                # df.to_csv('/Users/xigua/Downloads/mysql.csv', index=False, header=True, encoding='utf-8_sig')
-                # df.to_excel('/Users/xigua/Downloads/mysql.xlsx', index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
                 df.to_sql(
                     name=table_name,
                     con=engine,
                     if_exists='append',
                     index=False,
-                    chunksize=1000
+                    chunksize=1000,
+                    method='multi'
                 )
-                if reset_id:
-                    pass
                 connection.commit()  # 提交事务
                 connection.close()
                 return
@@ -889,13 +687,15 @@ class MysqlUpload:
             if move_insert and '日期' in df.columns.tolist():
                 # 移除数据
                 dates = df['日期'].values.tolist()
-                # logger.info(dates)
                 dates = [pd.to_datetime(item) for item in dates]  # 需要先转换类型才能用 min, max
                 start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
                 end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
 
-                sql = f"DELETE FROM `{table_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
-                cursor.execute(sql)
+                delete_sql = f"""
+                                DELETE FROM `{table_name}` 
+                                WHERE 日期 BETWEEN %s AND %s
+                            """
+                cursor.execute(delete_sql, (start_date, end_date))
                 connection.commit()
 
                 # 插入数据
@@ -906,7 +706,8 @@ class MysqlUpload:
                     con=engine,
                     if_exists='append',
                     index=False,
-                    chunksize=1000
+                    chunksize=1000,
+                    method='multi'
                 )
                 return
 
@@ -915,236 +716,78 @@ class MysqlUpload:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
                 try:
-                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
-                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
-                    condition = []
+                    # 预处理数据：转换非字符串类型
+                    processed_data = {}
                     for k, v in data.items():
-                        condition += [f'`{k}` = "{v}"']
-                    condition = ' AND '.join(condition)  # 构建查询条件
-                    # logger.info(condition)
-
-                    if drop_duplicates:  # 查重插入
-                        sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
-                        # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                        cursor.execute(sql)
-                        result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
-                        # logger.info(result)
-                        if not result:  # 数据不存在则插入
-                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
-                            # logger.info(sql)
-                            cursor.execute(sql)
-                        # else:
-                        #     logger.info(f'重复数据不插入: {condition[:50]}...')
-                    elif icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
-                        """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列必须是数据表中唯一主键，值不会发生变化且不会重复，否则可能产生覆盖情况 """
-                        sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                        cursor.execute(sql, (db_name, {table_name}))
-                        columns = cursor.fetchall()
-                        cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
-                        update_col = [item for item in cols_exist if
-                                      item not in icm_update and item != 'id']  # 除了主键外的其他列
-
-                        # unique_keys 示例: `日期`, `余额`
-                        unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
-                        condition = []
-                        for up_col in icm_update:
-                            condition += [f'`{up_col}` = "{data[up_col]}"']
-                        condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
-                        sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
-                        # logger.info(sql)
-                        # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                        cursor.execute(sql)
-                        results = cursor.fetchall()  # results 是数据库取出的数据
-                        if results:  # 有数据返回，再进行增量检查
-                            for result in results:  # results 是数据库数据, data 是传进来的数据
-                                change_col = []  # 发生变化的列名
-                                change_values = []  # 发生变化的数据
-                                for col in update_col:
-                                    # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
-                                    df_value = str(data[col])
-                                    mysql_value = str(result[col])
-                                    if '.' in df_value:
-                                        df_value = re.sub(r'0+$', '', df_value)
-                                        df_value = re.sub(r'\.$', '', df_value)
-                                    if '.' in mysql_value:
-                                        mysql_value = re.sub(r'0+$', '', mysql_value)
-                                        mysql_value = re.sub(r'\.$', '', mysql_value)
-                                    if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
-                                        # logger.info(f'{data['日期']}{data['商品id']}{col} 列的值有变化，{str(data[col])}  !=  {str(result[col])}')
-                                        change_values += [f"`{col}` = \"{str(data[col])}\""]
-                                        change_col.append(col)
-                                not_change_col = [item for item in update_col if item not in change_col]
-                                # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
-                                if change_values:  # change_values 有数据返回，表示值需要更新
-                                    if not_change_col:
-                                        not_change_values = [f'`{col}` = "{str(data[col])}"' for col in not_change_col]
-                                        not_change_values = ' AND '.join(
-                                            not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
-                                        # logger.info(change_values, not_change_values)
-                                        condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
-                                    change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
-                                    sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
-                                    # logger.info(sql)
-                                    cursor.execute(sql)
-                        else:  # 没有数据返回，则直接插入数据
-                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                            cursor.execute(sql)
+                        if isinstance(v, (int, float)):
+                            processed_data[k] = float(v)
+                        elif isinstance(v, pd.Timestamp):
+                            processed_data[k] = v.strftime('%Y-%m-%d')
+                        else:
+                            processed_data[k] = str(v)
+
+                    # 构建基础SQL要素
+                    columns = [f'`{k}`' for k in processed_data.keys()]
+                    placeholders = ', '.join(['%s'] * len(processed_data))
+                    values = list(processed_data.values())
+
+                    # 构建基本INSERT语句
+                    insert_sql = f"INSERT INTO `{table_name}` ({', '.join(columns)}) VALUES ({placeholders})"
+
+                    if icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
+                        # 获取数据表结构
+                        cursor.execute(
+                            "SELECT COLUMN_NAME FROM information_schema.columns "
+                            "WHERE table_schema = %s AND table_name = %s",
+                            (db_name, table_name)
+                        )
+                        cols_exist = [row['COLUMN_NAME'] for row in cursor.fetchall()]
+                        update_columns = [col for col in cols_exist if col not in icm_update and col != 'id']
+
+                        # 构建WHERE条件
+                        where_conditions = []
+                        where_values = []
+                        for col in icm_update:
+                            where_conditions.append(f"`{col}` = %s")
+                            where_values.append(processed_data[col])
+
+                        # 查询现有数据
+                        select_sql = f"SELECT {', '.join([f'`{col}`' for col in update_columns])} " \
+                                     f"FROM `{table_name}` WHERE {' AND '.join(where_conditions)}"
+                        cursor.execute(select_sql, where_values)
+                        existing_data = cursor.fetchone()
+
+                        if existing_data:
+                            # 比较并构建更新语句
+                            update_set = []
+                            update_values = []
+                            for col in update_columns:
+                                db_value = existing_data[col]
+                                new_value = processed_data[col]
+
+                                # 处理数值类型的精度差异
+                                if isinstance(db_value, float) and isinstance(new_value, float):
+                                    if not math.isclose(db_value, new_value, rel_tol=1e-9):
+                                        update_set.append(f"`{col}` = %s")
+                                        update_values.append(new_value)
+                                elif db_value != new_value:
+                                    update_set.append(f"`{col}` = %s")
+                                    update_values.append(new_value)
+
+                            if update_set:
+                                update_sql = f"UPDATE `{table_name}` SET {', '.join(update_set)} " \
+                                             f"WHERE {' AND '.join(where_conditions)}"
+                                cursor.execute(update_sql, update_values + where_values)
+                        else:
+                            cursor.execute(insert_sql, values)
                     else:
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
-                        cursor.execute(sql)
+                        # 普通插入
+                        cursor.execute(insert_sql, values)
                 except Exception as e:
                     pass
-
-            if reset_id:
-                pass
         connection.commit()  # 提交事务
         connection.close()
 
-    @try_except
-    def read_doc_data(self, table_name, db_name='pdf文件', column='文件名', filename=None, save_path='/Users/xigua/Downloads'):
-        """
-        db_name:
-        table_name:
-        column: 读取哪一列
-        filename: 文件名称
-        save_path: 保存位置
-        """
-        if not filename:
-            logger.info(f'未指定文件名: filename')
-            return
-        # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
-        if not connection:
-            return
-        # try:
-        with connection.cursor() as cursor:
-            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
-            database_exists = cursor.fetchone()
-            if not database_exists:
-                logger.info(f"Database {db_name} 数据库不存在")
-                return
-        self.config.update({'database': db_name})
-        # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
-        if not connection:
-            return
-        with connection.cursor() as cursor:
-            # 1. 查询表
-            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
-            cursor.execute(sql, (table_name))
-            if not cursor.fetchone():
-                logger.info(f'{table_name} -> 数据表不存在')
-                return
-
-            # 读取数据
-            condition = f'`{column}` = "{filename}"'
-            sql = f"SELECT `{column}`, `数据主体` FROM `{table_name}` WHERE {condition}"
-            cursor.execute(sql)
-            results = cursor.fetchall()
-            if results:
-                for result in results:
-                    # 将二进制数据写入到文件
-                    with open(os.path.join(save_path, filename), 'wb') as f:
-                        f.write(result['数据主体'])
-                        logger.info(f'写入本地文件: ({self.host}:{self.port}) {db_name}/{table_name} -> {os.path.join(save_path, filename)}')
-        connection.close()
-
-    def read_mysql(self, table_name, start_date, end_date, db_name='远程数据源', date_name='日期'):
-        """ 读取指定数据表，可指定日期范围，返回结果: df """
-        start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
-        end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
-        df = pd.DataFrame()
-
-        # connection = pymysql.connect(**self.config)  # 连接数据库
-        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
-        if not connection:
-            return
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
-                database_exists = cursor.fetchone()
-                if not database_exists:
-                    logger.info(f"Database {db_name} 数据库不存在")
-                    return df
-                else:
-                    logger.info(f'mysql 正在查询表: {table_name}, 范围: {start_date}~{end_date}')
-        except:
-            return df
-        finally:
-            connection.close()  # 断开连接
-
-        before_time = time.time()
-        # 读取数据
-        self.config.update({'database': db_name})
-        # connection = pymysql.connect(**self.config)  # 重新连接数据库
-        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
-        if not connection:
-            return
-        try:
-            with connection.cursor() as cursor:
-                # 获取指定日期范围的数据
-                sql = f"SELECT * FROM `{db_name}`.`{table_name}` WHERE `{date_name}` BETWEEN '%s' AND '%s'" % (start_date, end_date)
-                cursor.execute(sql)
-                rows = cursor.fetchall()  # 获取查询结果
-                columns = [desc[0] for desc in cursor.description]
-                df = pd.DataFrame(rows, columns=columns)  # 转为 df
-        except Exception as e:
-            logger.error(f'{e} {db_name} -> {table_name} 表不存在')
-            return df
-        finally:
-            connection.close()
-
-        if len(df) == 0:
-            logger.info(f'database: {db_name}, table: {table_name} 查询的数据为空')
-        else:
-            cost_time = int(time.time() - before_time)
-            if cost_time < 1:
-                cost_time = round(time.time() - before_time, 2)
-            logger.info(f'mysql ({self.host}) 表: {table_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
-        return df
-
-    def upload_pandas(self, update_path, db_name, days=None):
-        """
-        专门用来上传 pandas数据源的全部文件
-        db_name: 数据库名: pandas数据源
-        update_path: pandas数据源所在路径
-        days: 更新近期数据，单位: 天, 不设置则全部更新
-        """
-        if days:
-            today = datetime.date.today()
-            start_date = pd.to_datetime(today - datetime.timedelta(days=days))
-        else:
-            start_date = pd.to_datetime('2000-01-01')
-
-        root_files = os.listdir(update_path)
-        for root_file in root_files:
-            if '其他数据' in root_file or '年.csv' in root_file or '京东数据集' in root_file:
-                continue  # 跳过的文件夹
-            f_path = os.path.join(update_path, root_file)
-
-            if os.path.isdir(f_path):
-                for root, dirs, files in os.walk(f_path, topdown=False):
-                    for name in files:
-                        if name.endswith('.csv') and 'baidu' not in name:
-                            df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                            if '日期' in df.columns.tolist():
-                                df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x) if x else x)
-                                df = df[df['日期'] >= start_date]
-                            if len(df) == 0:
-                                continue
-                            self.df_to_mysql(df=df, db_name=db_name, table_name=root_file)
-            elif os.path.isfile(f_path):
-                if f_path.endswith('.csv') and 'baidu' not in f_path:
-                    df = pd.read_csv(f_path, encoding='utf-8_sig', header=0, na_filter=False)
-                    if '日期' not in df.columns.tolist():
-                        df['日期'] = df['日期'].apply(lambda x: pd.to_datetime(x) if x else x)
-                        df = df[df['日期'] >= start_date]
-                    if len(df) == 0:
-                        continue
-                    table = f'{os.path.splitext(root_file)[0]}_f'  # 这里定义了文件表会加 _f 后缀
-                    self.df_to_mysql(df=df, db_name=db_name, table_name=table)
-
 
 class OptimizeDatas:
     """