mdbq 3.9.0__py3-none-any.whl → 3.9.1__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.9.0'
+VERSION = '3.9.1'

mdbq/mysql/mysql.py

@@ -11,10 +11,8 @@ from sqlalchemy import create_engine
 import os
 import logging
 from mdbq.other import otk
-
 from dbutils.pooled_db import PooledDB
 from typing import Union, List, Dict, Optional, Any, Tuple
-
 warnings.filterwarnings('ignore')
 """
 建表流程:
@@ -59,7 +57,11 @@ class MySQLUploader:
             log_level: str = 'ERROR',
             max_retries: int = 10,
             retry_interval: int = 10,
-            pool_size: int = 5
+            pool_size: int = 5,
+            connect_timeout: int = 10,
+            read_timeout: int = 30,
+            write_timeout: int = 30,
+            ssl: Optional[Dict] = None
     ):
         """
         初始化MySQL上传工具
@@ -75,6 +77,10 @@ class MySQLUploader:
         :param max_retries: 最大重试次数，默认为10
         :param retry_interval: 重试间隔(秒)，默认为10
         :param pool_size: 连接池大小，默认为5
+        :param connect_timeout: 连接超时(秒)，默认为10
+        :param read_timeout: 读取超时(秒)，默认为30
+        :param write_timeout: 写入超时(秒)，默认为30
+        :param ssl: SSL配置字典，默认为None
         """
         self.username = username
         self.password = password
@@ -82,37 +88,90 @@ class MySQLUploader:
         self.port = port
         self.charset = charset
         self.collation = collation
-        self.max_retries = max_retries
-        self.retry_interval = retry_interval
-        self.pool_size = pool_size
+        self.max_retries = max(max_retries, 1)  # 至少重试1次
+        self.retry_interval = max(retry_interval, 1)  # 至少间隔1秒
+        self.pool_size = max(pool_size, 1)  # 至少1个连接
+        self.connect_timeout = connect_timeout
+        self.read_timeout = read_timeout
+        self.write_timeout = write_timeout
+        self.ssl = ssl
+        self._prepared_statements = {}  # 预处理语句缓存
+        self._max_cached_statements = 100  # 最大缓存语句数
 
         # 初始化日志
         if enable_logging:
             self._init_logging(log_level)
+        else:
+            self.logger = None
 
         # 创建连接池
         self.pool = self._create_connection_pool()
 
     def _init_logging(self, log_level: str):
         """初始化日志配置"""
+        valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+        level = log_level.upper() if log_level.upper() in valid_levels else 'ERROR'
+
         logging.basicConfig(
-            level=getattr(logging, log_level.upper(), logging.ERROR),
-            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+            level=getattr(logging, level),
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[logging.StreamHandler()]
         )
         self.logger = logging.getLogger('MySQLUploader')
 
-    def _create_connection_pool(self):
+    def _create_connection_pool(self) -> PooledDB:
         """创建数据库连接池"""
-        return PooledDB(
-            creator=pymysql,
-            host=self.host,
-            port=self.port,
-            user=self.username,
-            password=self.password,
-            charset=self.charset,
-            maxconnections=self.pool_size,
-            cursorclass=pymysql.cursors.DictCursor
-        )
+        pool_params = {
+            'creator': pymysql,
+            'host': self.host,
+            'port': self.port,
+            'user': self.username,
+            'password': self.password,
+            'charset': self.charset,
+            'cursorclass': pymysql.cursors.DictCursor,
+            'maxconnections': self.pool_size,
+            'ping': 7,  # 连接检查
+            'connect_timeout': self.connect_timeout,
+            'read_timeout': self.read_timeout,
+            'write_timeout': self.write_timeout,
+            'autocommit': False
+        }
+
+        if self.ssl:
+            required_keys = {'ca', 'cert', 'key'}
+            if not all(k in self.ssl for k in required_keys):
+                raise ValueError("SSL配置必须包含ca、cert和key")
+            pool_params['ssl'] = {
+                'ca': self.ssl['ca'],
+                'cert': self.ssl['cert'],
+                'key': self.ssl['key'],
+                'check_hostname': self.ssl.get('check_hostname', False)
+            }
+
+        try:
+            pool = PooledDB(**pool_params)
+            return pool
+        except Exception as e:
+            if self.logger:
+                self.logger.error("连接池创建失败: %s", str(e))
+            raise ConnectionError(f"连接池创建失败: {str(e)}")
+
+    def _validate_datetime(self, value):
+        formats = [
+            '%Y-%m-%d %H:%M:%S',
+            '%Y-%m-%d',
+            '%Y/%m/%d %H:%M:%S',
+            '%Y/%m/%d',
+            '%Y%m%d',
+            '%Y-%m-%dT%H:%M:%S',  # ISO格式
+            '%Y-%m-%d %H:%M:%S.%f'  # 带毫秒
+        ]
+        for fmt in formats:
+            try:
+                return datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
+            except ValueError:
+                continue
+        raise ValueError(f"无效的日期格式: {value}")
 
     def _validate_identifier(self, identifier: str) -> str:
         """
@@ -121,14 +180,31 @@ class MySQLUploader:
 
         :param identifier: 要验证的标识符
         :return: 清理后的安全标识符
+        :raises ValueError: 如果标识符无效
         """
         if not identifier or not isinstance(identifier, str):
-            raise ValueError(f"Invalid identifier: {identifier}")
+            error_msg = f"无效的标识符: {identifier}"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
-        # 移除可能有害的字符
-        cleaned = re.sub(r'[^a-zA-Z0-9_$]', '', identifier)
+        # 移除可能有害的字符，只保留字母、数字、下划线和美元符号
+        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '', identifier)
         if not cleaned:
-            raise ValueError(f"Invalid identifier after cleaning: {identifier}")
+            error_msg = f"无法清理异常标识符: {identifier}"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # 检查是否为MySQL保留字
+        mysql_keywords = {
+            'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
+            'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
+        }
+        if cleaned.lower() in mysql_keywords:
+            if self.logger:
+                self.logger.warning("存在MySQL保留字: %s", cleaned)
+            return f"`{cleaned}`"
 
         return cleaned
 
@@ -139,25 +215,42 @@ class MySQLUploader:
         :param value: 要验证的值
         :param column_type: 列的数据类型
         :return: 清理后的值
+        :raises ValueError: 如果值转换失败
         """
         if value is None:
             return None
 
         try:
-            if 'int' in column_type.lower():
+            column_type_lower = column_type.lower()
+
+            if 'int' in column_type_lower:
                 return int(value) if value is not None else None
-            elif 'float' in column_type.lower() or 'double' in column_type.lower() or 'decimal' in column_type.lower():
+            elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
                 return float(value) if value is not None else None
-            elif 'date' in column_type.lower() or 'time' in column_type.lower():
+            elif '日期' in column_type_lower or 'time' in column_type_lower:
                 if isinstance(value, (datetime.datetime, pd.Timestamp)):
                     return value.strftime('%Y-%m-%d %H:%M:%S')
+                elif isinstance(value, str):
+                    try:
+                        return self._validate_datetime(value)  # 使用专门的日期验证方法
+                    except ValueError as e:
+                        raise ValueError(f"无效日期格式: {value} - {str(e)}")
                 return str(value)
-            elif 'char' in column_type.lower() or 'text' in column_type.lower():
+            elif 'char' in column_type_lower or 'text' in column_type_lower:
+                # 防止SQL注入
+                if isinstance(value, str):
+                    return value.replace('\\', '\\\\').replace("'", "\\'")
                 return str(value)
+            elif 'json' in column_type_lower:
+                import json
+                return json.dumps(value) if value is not None else None
             else:
                 return value
         except (ValueError, TypeError) as e:
-            raise ValueError(f"Failed to convert value {value} to type {column_type}: {str(e)}")
+            error_msg = f"数据类型转换异常 {value} to type {column_type}: {str(e)}"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
     def _execute_with_retry(self, func, *args, **kwargs):
         """
@@ -167,6 +260,7 @@ class MySQLUploader:
         :param args: 位置参数
         :param kwargs: 关键字参数
         :return: 函数执行结果
+        :raises Exception: 如果所有重试都失败
         """
 
         @wraps(func)
@@ -174,41 +268,95 @@ class MySQLUploader:
             last_exception = None
             for attempt in range(self.max_retries):
                 try:
-                    return func(*args, **kwargs)
-                except pymysql.OperationalError as e:
+                    result = func(*args, **kwargs)
+                    if attempt > 0 and self.logger:
+                        self.logger.info("Operation succeeded after %d retries", attempt)
+                    return result
+                except (pymysql.OperationalError, pymysql.err.MySQLError) as e:
                     last_exception = e
                     if attempt < self.max_retries - 1:
-                        time.sleep(self.retry_interval)
+                        wait_time = self.retry_interval * (attempt + 1)
+                        if self.logger:
+                            self.logger.warning(
+                                "尝试 %d/%d 失败: %s. %d秒后重试...",
+                                attempt + 1, self.max_retries, str(e), wait_time
+                            )
+                        time.sleep(wait_time)
                         # 尝试重新连接
-                        self.pool = self._create_connection_pool()
+                        try:
+                            self.pool = self._create_connection_pool()
+                        except Exception as reconnect_error:
+                            if self.logger:
+                                self.logger.error("重连失败: %s", str(reconnect_error))
                         continue
-            raise last_exception if last_exception else Exception("Unknown error occurred")
+                    else:
+                        if self.logger:
+                            self.logger.error(
+                                "Operation failed after %d attempts. Last error: %s",
+                                self.max_retries, str(e)
+                            )
+                except pymysql.IntegrityError as e:
+                    # 完整性错误通常不需要重试
+                    if self.logger:
+                        self.logger.error("完整性约束错误: %s", str(e))
+                    raise e
+                except Exception as e:
+                    last_exception = e
+                    if self.logger:
+                        self.logger.error("发生意外错误: %s", str(e))
+                    break
+
+            raise last_exception if last_exception else Exception("发生未知错误")
 
         return wrapper(*args, **kwargs)
 
     def _get_connection(self):
         """从连接池获取连接"""
-        return self.pool.connection()
+        try:
+            conn = self.pool.connection()
+            if self.logger:
+                self.logger.debug("成功获取数据库连接")
+            return conn
+        except Exception as e:
+            if self.logger:
+                self.logger.error("连接数据库失败: %s", str(e))
+            raise ConnectionError(f"连接数据库失败: {str(e)}")
 
     def _check_database_exists(self, db_name: str) -> bool:
         """检查数据库是否存在"""
         db_name = self._validate_identifier(db_name)
         sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
 
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql, (db_name,))
-                return bool(cursor.fetchone())
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql, (db_name,))
+                    exists = bool(cursor.fetchone())
+                    if self.logger:
+                        self.logger.debug("数据库 %s 已存在: %s", db_name, exists)
+                    return exists
+        except Exception as e:
+            if self.logger:
+                self.logger.error("检查数据库是否存在时出错: %s", str(e))
+            raise
 
     def _create_database(self, db_name: str):
         """创建数据库"""
         db_name = self._validate_identifier(db_name)
         sql = f"CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}"
 
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql)
-            conn.commit()
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                conn.commit()
+                if self.logger:
+                    self.logger.info("数据库 %s 创建成功", db_name)
+        except Exception as e:
+            if self.logger:
+                self.logger.error("无法创建数据库 %s: %s", db_name, str(e))
+            conn.rollback()
+            raise
 
     def _check_table_exists(self, db_name: str, table_name: str) -> bool:
         """检查表是否存在"""
@@ -220,10 +368,16 @@ class MySQLUploader:
         WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
         """
 
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql, (db_name, table_name))
-                return bool(cursor.fetchone())
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql, (db_name, table_name))
+                    exists = bool(cursor.fetchone())
+                    return exists
+        except Exception as e:
+            if self.logger:
+                self.logger.error("检查数据表是否存在时发生未知错误: %s", str(e))
+            raise
 
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
         """获取表的列名和数据类型"""
@@ -233,69 +387,21 @@ class MySQLUploader:
         SELECT COLUMN_NAME, DATA_TYPE 
         FROM INFORMATION_SCHEMA.COLUMNS 
         WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        ORDER BY ORDINAL_POSITION
         """
 
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql, (db_name, table_name))
-                return {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
-
-    def _create_table(
-            self,
-            db_name: str,
-            table_name: str,
-            columns: Dict[str, str],
-            primary_keys: Optional[List[str]] = None,
-            date_column: Optional[str] = None
-    ):
-        """
-        创建数据表
-
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns: 列名和数据类型字典 {列名: 数据类型}
-        :param primary_keys: 主键列列表
-        :param date_column: 日期列名，如果存在将设置为索引
-        """
-        db_name = self._validate_identifier(db_name)
-        table_name = self._validate_identifier(table_name)
-
-        if not columns:
-            raise ValueError("No columns specified for table creation")
-
-        # 构建列定义SQL
-        column_defs = []
-        for col_name, col_type in columns.items():
-            safe_col_name = self._validate_identifier(col_name)
-            col_def = f"`{safe_col_name}` {col_type}"
-            column_defs.append(col_def)
-
-        # 添加主键定义
-        primary_key_sql = ""
-        if primary_keys:
-            safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
-            primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
-
-        # 构建完整SQL
-        sql = f"""
-        CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
-            {','.join(column_defs)}
-            {primary_key_sql}
-        ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
-        """
-
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql)
-
-            # 如果存在日期列，添加索引
-            if date_column and date_column in columns:
-                safe_date_col = self._validate_identifier(date_column)
-                index_sql = f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
+        try:
+            with self._get_connection() as conn:
                 with conn.cursor() as cursor:
-                    cursor.execute(index_sql)
-
-            conn.commit()
+                    cursor.execute(sql, (db_name, table_name))
+                    columns = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
+                    if self.logger:
+                        self.logger.debug("获取表 %s.%s 的列信息: %s", db_name, table_name, columns)
+                    return columns
+        except Exception as e:
+            if self.logger:
+                self.logger.error("无法获取表列信息: %s", str(e))
+            raise
 
     def _prepare_data(
             self,
@@ -310,31 +416,51 @@ class MySQLUploader:
         :param columns: 列名和数据类型字典 {列名: 数据类型}
         :param allow_null: 是否允许空值
         :return: 准备好的数据列表
+        :raises ValueError: 如果数据验证失败
         """
         # 统一数据格式为字典列表
         if isinstance(data, pd.DataFrame):
-            data = data.to_dict('records')
+            try:
+                data = data.replace({pd.NA: None}).to_dict('records')
+            except Exception as e:
+                if self.logger:
+                    self.logger.error("Failed to convert DataFrame to dict: %s", str(e))
+                raise ValueError(f"Failed to convert DataFrame to dict: {str(e)}")
         elif isinstance(data, dict):
             data = [data]
-
-        if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
-            raise ValueError("Data must be a dict, list of dicts, or DataFrame")
+        elif not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
+            error_msg = "Data must be a dict, list of dicts, or DataFrame"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
         prepared_data = []
-        for row in data:
+        for row_idx, row in enumerate(data, 1):
             prepared_row = {}
             for col_name, col_type in columns.items():
+                # 跳过id列，不允许外部传入id
+                if col_name.lower() == 'id':
+                    continue
+
                 if col_name not in row:
                     if not allow_null:
-                        raise ValueError(f"Missing required column '{col_name}' in data")
+                        error_msg = f"Row {row_idx}: Missing required column '{col_name}' in data"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise ValueError(error_msg)
                     prepared_row[col_name] = None
                 else:
                     try:
                         prepared_row[col_name] = self._validate_value(row[col_name], col_type)
                     except ValueError as e:
-                        raise ValueError(f"Error in column '{col_name}': {str(e)}")
+                        error_msg = f"Row {row_idx}, column '{col_name}': {str(e)}"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise ValueError(error_msg)
             prepared_data.append(prepared_row)
 
+        if self.logger:
+            self.logger.debug("已准备 %d 行数据", len(prepared_data))
         return prepared_data
 
     def _get_partition_table_name(self, table_name: str, date_value: str, partition_by: str) -> str:
@@ -345,6 +471,7 @@ class MySQLUploader:
         :param date_value: 日期值
         :param partition_by: 分表方式 ('year' 或 'month')
         :return: 分表名称
+        :raises ValueError: 如果日期格式无效或分表方式无效
         """
         try:
             date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
@@ -352,92 +479,141 @@ class MySQLUploader:
             try:
                 date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
             except ValueError:
-                raise ValueError(f"Invalid date format: {date_value}")
+                error_msg = f"无效的日期格式: {date_value}"
+                if self.logger:
+                    self.logger.error("无效的日期格式: %s", date_value)
+                raise ValueError(error_msg)
 
         if partition_by == 'year':
             return f"{table_name}_{date_obj.year}"
         elif partition_by == 'month':
             return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
         else:
-            raise ValueError("partition_by must be 'year' or 'month'")
+            error_msg = "partition_by must be 'year' or 'month'"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
-    def _insert_data(
+    def _create_table(
             self,
             db_name: str,
             table_name: str,
-            data: List[Dict],
             columns: Dict[str, str],
-            check_duplicate: bool = False,
-            duplicate_columns: Optional[List[str]] = None,
-            batch_size: int = 1000
+            primary_keys: Optional[List[str]] = None,
+            date_column: Optional[str] = None,
+            indexes: Optional[List[str]] = None,
+            unique_columns: Optional[List[str]] = None
     ):
         """
-        插入数据到表中
+        创建数据表
 
         :param db_name: 数据库名
         :param table_name: 表名
-        :param data: 要插入的数据
-        :param columns: 列名和数据类型字典
-        :param check_duplicate: 是否检查重复
-        :param duplicate_columns: 用于检查重复的列列表
-        :param batch_size: 批量插入的大小
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param date_column: 日期列名，如果存在将设置为索引
+        :param indexes: 需要创建索引的列列表
+        :param unique_columns: 需要创建唯一索引的列列表
         """
         db_name = self._validate_identifier(db_name)
         table_name = self._validate_identifier(table_name)
 
-        if not data:
-            return
+        if not columns:
+            error_msg = "No columns specified for table creation"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
-        # 获取所有列名
-        all_columns = list(columns.keys())
-        safe_columns = [self._validate_identifier(col) for col in all_columns]
-        placeholders = ','.join(['%s'] * len(safe_columns))
+        # 构建列定义SQL
+        column_defs = ["`id` INT NOT NULL AUTO_INCREMENT"]
 
-        # 构建INSERT SQL
-        if check_duplicate:
-            if duplicate_columns:
-                # 只检查指定列的重复
-                dup_columns = [self._validate_identifier(col) for col in duplicate_columns]
-            else:
-                # 检查所有列的重复
-                dup_columns = safe_columns
+        # 添加其他列定义
+        for col_name, col_type in columns.items():
+            # 跳过id列，因为已经在前面添加了
+            if col_name.lower() == 'id':
+                continue
+            safe_col_name = self._validate_identifier(col_name)
+            col_def = f"`{safe_col_name}` {col_type}"
 
-            # 构建ON DUPLICATE KEY UPDATE子句
-            update_clause = ','.join([f"`{col}`=VALUES(`{col}`)" for col in safe_columns])
+            # 添加NOT NULL约束
+            if not col_type.lower().startswith('json'):
+                col_def += " NOT NULL"
 
-            sql = f"""
-            INSERT INTO `{db_name}`.`{table_name}` 
-            (`{'`,`'.join(safe_columns)}`) 
-            VALUES ({placeholders})
-            ON DUPLICATE KEY UPDATE {update_clause}
-            """
+            column_defs.append(col_def)
+
+        # 添加主键定义
+        if primary_keys:
+            # 确保id在主键中
+            if 'id' not in [pk.lower() for pk in primary_keys]:
+                primary_keys = ['id'] + primary_keys
         else:
-            sql = f"""
-            INSERT INTO `{db_name}`.`{table_name}` 
-            (`{'`,`'.join(safe_columns)}`) 
-            VALUES ({placeholders})
-            """
+            # 如果没有指定主键，则使用id作为主键
+            primary_keys = ['id']
 
-        # 分批插入数据
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                for i in range(0, len(data), batch_size):
-                    batch = data[i:i + batch_size]
-                    # 准备批量数据
-                    values = []
-                    for row in batch:
-                        row_values = []
-                        for col in all_columns:
-                            row_values.append(row.get(col))
-                        values.append(row_values)
+        # 添加主键定义
+        safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
+        primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
 
-                    # 执行批量插入
-                    try:
-                        cursor.executemany(sql, values)
-                        conn.commit()
-                    except Exception as e:
-                        conn.rollback()
-                        raise e
+        # 添加唯一索引定义
+        unique_index_sql = ""
+        if unique_columns:
+            for col in unique_columns:
+                if col.lower() != 'id' and col in columns:
+                    safe_col = self._validate_identifier(col)
+                    unique_index_sql += f", UNIQUE KEY `uk_{safe_col}` (`{safe_col}`)"
+
+        # 构建完整SQL
+        sql = f"""
+        CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
+            {','.join(column_defs)}
+            {primary_key_sql}
+            {unique_index_sql}
+        ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
+        """
+
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                    if self.logger:
+                        self.logger.info("表 %s.%s 创建成功", db_name, table_name)
+
+                # 添加普通索引
+                index_statements = []
+
+                # 日期列索引
+                if date_column and date_column in columns:
+                    safe_date_col = self._validate_identifier(date_column)
+                    index_statements.append(
+                        f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
+                    )
+
+                # 其他索引
+                if indexes:
+                    for idx_col in indexes:
+                        if idx_col in columns:
+                            safe_idx_col = self._validate_identifier(idx_col)
+                            index_statements.append(
+                                f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)"
+                            )
+
+                # 执行所有索引创建语句
+                if index_statements:
+                    with conn.cursor() as cursor:
+                        for stmt in index_statements:
+                            cursor.execute(stmt)
+                            if self.logger:
+                                self.logger.debug("Executed index statement: %s", stmt)
+
+                conn.commit()
+                if self.logger:
+                    self.logger.info("All indexes created successfully for %s.%s", db_name, table_name)
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error("创建表 %s.%s 失败: %s", db_name, table_name, str(e))
+            conn.rollback()
+            raise
 
     def upload_data(
             self,
@@ -451,7 +627,9 @@ class MySQLUploader:
             allow_null: bool = False,
             partition_by: Optional[str] = None,
             partition_date_column: str = '日期',
-            auto_create: bool = True
+            auto_create: bool = True,
+            replace: bool = False,
+            indexes: Optional[List[str]] = None
     ):
         """
         上传数据到数据库
@@ -465,15 +643,30 @@ class MySQLUploader:
         :param duplicate_columns: 用于检查重复的列列表，如果不指定则使用所有列
         :param allow_null: 是否允许空值，默认为False
         :param partition_by: 分表方式 ('year' 或 'month')，默认为None不分表
-        :param partition_date_column: 用于分表的日期列名，默认为'日期'
+        :param partition_date_column: 用于分表的日期列名，默认为'date'
         :param auto_create: 是否自动创建不存在的数据库或表，默认为True
+        :param replace: 是否使用REPLACE代替INSERT，默认为False
+        :param indexes: 需要创建索引的列列表
+        :raises ValueError: 如果参数无效或操作失败
         """
+        if self.logger:
+            self.logger.info(
+                "开始上传数据到 %s.%s (分表方式=%s, 替换模式=%s)",
+                db_name, table_name, partition_by, replace
+            )
+
         # 验证参数
         if not columns:
-            raise ValueError("Columns specification is required")
+            error_msg = "Columns specification is required"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
         if partition_by and partition_by not in ['year', 'month']:
-            raise ValueError("partition_by must be 'year', 'month' or None")
+            error_msg = "分表方式必须是 'year' 或 'month'"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
         # 准备数据
         prepared_data = self._prepare_data(data, columns, allow_null)
@@ -483,7 +676,16 @@ class MySQLUploader:
             if auto_create:
                 self._create_database(db_name)
             else:
-                raise ValueError(f"Database '{db_name}' does not exist")
+                error_msg = f"Database '{db_name}' does not exist"
+                if self.logger:
+                    self.logger.error(error_msg)
+                raise ValueError(error_msg)
+
+        # 确定唯一索引列
+        unique_columns = None
+        if check_duplicate:
+            unique_columns = duplicate_columns if duplicate_columns else [col for col in columns.keys() if
+                                                                          col.lower() != 'id']
 
         # 处理分表逻辑
         if partition_by:
@@ -491,7 +693,10 @@ class MySQLUploader:
             partitioned_data = {}
             for row in prepared_data:
                 if partition_date_column not in row:
-                    raise ValueError(f"Partition date column '{partition_date_column}' not found in data")
+                    error_msg = f"异常缺失列 '{partition_date_column}'"
+                    if self.logger:
+                        self.logger.error(error_msg)
+                    raise ValueError(error_msg)
                 part_table = self._get_partition_table_name(table_name, str(row[partition_date_column]), partition_by)
                 if part_table not in partitioned_data:
                     partitioned_data[part_table] = []
@@ -502,14 +707,22 @@ class MySQLUploader:
                 self._upload_to_table(
                     db_name, part_table, part_data, columns,
                     primary_keys, check_duplicate, duplicate_columns,
-                    allow_null, auto_create, partition_date_column
+                    allow_null, auto_create, partition_date_column,
+                    replace, indexes, unique_columns
                 )
         else:
             # 不分表，直接上传
             self._upload_to_table(
                 db_name, table_name, prepared_data, columns,
                 primary_keys, check_duplicate, duplicate_columns,
-                allow_null, auto_create, partition_date_column
+                allow_null, auto_create, partition_date_column,
+                replace, indexes, unique_columns
+            )
+
+        if self.logger:
+            self.logger.info(
+                "成功上传 %d 行数据到 %s.%s",
+                len(prepared_data), db_name, table_name
             )
 
     def _upload_to_table(
@@ -523,41 +736,178 @@ class MySQLUploader:
             duplicate_columns: Optional[List[str]],
             allow_null: bool,
             auto_create: bool,
-            date_column: Optional[str]
+            date_column: Optional[str],
+            replace: bool,
+            indexes: Optional[List[str]],
+            unique_columns: Optional[List[str]] = None
     ):
         """实际执行表上传的内部方法"""
         # 检查表是否存在
         if not self._check_table_exists(db_name, table_name):
             if auto_create:
-                self._create_table(db_name, table_name, columns, primary_keys, date_column)
+                self._create_table(db_name, table_name, columns, primary_keys, date_column, indexes, unique_columns)
             else:
-                raise ValueError(f"Table '{db_name}.{table_name}' does not exist")
+                error_msg = f"Table '{db_name}.{table_name}' does not exist"
+                if self.logger:
+                    self.logger.error(error_msg)
+                raise ValueError(error_msg)
 
         # 获取表结构并验证
         table_columns = self._get_table_columns(db_name, table_name)
         if not table_columns:
-            raise ValueError(f"Failed to get columns for table '{db_name}.{table_name}'")
+            error_msg = f"Failed to get columns for table '{db_name}.{table_name}'"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
         # 验证数据列与表列匹配
         for col in columns:
             if col not in table_columns:
-                raise ValueError(f"Column '{col}' not found in table '{db_name}.{table_name}'")
+                error_msg = f"Column '{col}' not found in table '{db_name}.{table_name}'"
+                if self.logger:
+                    self.logger.error(error_msg)
+                raise ValueError(error_msg)
 
         # 插入数据
         self._insert_data(
             db_name, table_name, data, columns,
-            check_duplicate, duplicate_columns
+            check_duplicate, duplicate_columns,
+            replace=replace
         )
 
+    def _insert_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            batch_size: int = 1000,
+            replace: bool = False
+    ):
+        """
+        插入数据到表中
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要插入的数据
+        :param columns: 列名和数据类型字典
+        :param check_duplicate: 是否检查重复
+        :param duplicate_columns: 用于检查重复的列列表
+        :param batch_size: 批量插入的大小
+        :param replace: 是否使用REPLACE代替INSERT
+        :raises Exception: 如果插入失败
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not data:
+            if self.logger:
+                self.logger.warning("No data to insert into %s.%s", db_name, table_name)
+            return
+
+        # 获取所有列名
+        all_columns = [col for col in columns.keys() if col.lower() != 'id']
+        safe_columns = [self._validate_identifier(col) for col in all_columns]
+        placeholders = ','.join(['%s'] * len(safe_columns))
+
+        # 构建SQL语句
+        operation = "REPLACE" if replace else "INSERT IGNORE" if check_duplicate else "INSERT"
+
+        if check_duplicate and not replace:
+            # 当check_duplicate=True时，使用INSERT IGNORE来跳过重复记录
+            sql = f"""
+            {operation} INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            """
+        else:
+            sql = f"""
+            {operation} INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            """
+
+        if len(self._prepared_statements) >= self._max_cached_statements:
+            # 移除最旧的缓存
+            oldest_key = next(iter(self._prepared_statements))
+            del self._prepared_statements[oldest_key]
+
+        # 缓存预处理语句
+        cache_key = f"{db_name}.{table_name}.{operation}.{check_duplicate}"
+        if cache_key not in self._prepared_statements:
+            self._prepared_statements[cache_key] = sql
+            if self.logger:
+                self.logger.debug("已缓存预处理语句: %s", cache_key)
+
+        # 分批插入数据
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                for i in range(0, len(data), batch_size):
+                    batch = data[i:i + batch_size]
+                    # 准备批量数据
+                    values = []
+                    for row in batch:
+                        row_values = []
+                        for col in all_columns:
+                            row_values.append(row.get(col))
+                        values.append(row_values)
+
+                    # 执行批量插入
+                    try:
+                        start_time = time.time()
+                        cursor.executemany(sql, values)
+                        conn.commit()  # 每个批次提交一次
+                        if self.logger:
+                            self.logger.debug(
+                                "成功插入批次 %d-%d/%d 到 %s.%s, 耗时 %.2f 秒",
+                                i + 1, min(i + batch_size, len(data)), len(data),
+                                db_name, table_name, time.time() - start_time
+                            )
+                    except Exception as e:
+                        conn.rollback()
+                        error_msg = f"Failed to insert batch {i + 1}-{min(i + batch_size, len(data))}/{len(data)} into {db_name}.{table_name}: {str(e)}"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise Exception(error_msg)
+
     def close(self):
         """关闭连接池"""
-        self.pool.close()
+        if hasattr(self, 'pool') and self.pool:
+            try:
+                # 先关闭所有连接
+                while True:
+                    conn = getattr(self.pool, '_connections', None)
+                    if not conn or not conn.queue:
+                        break
+                    try:
+                        conn = self.pool.connection()
+                        conn.close()
+                    except:
+                        pass
+
+                # 然后关闭连接池
+                self.pool.close()
+                if self.logger:
+                    self.logger.info("连接池已成功关闭")
+            except Exception as e:
+                if self.logger:
+                    self.logger.error("关闭连接池失败: %s", str(e))
+                raise
+        self.pool = None
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
+        if exc_type is not None and self.logger:
+            self.logger.error(
+                "Exception occurred: %s: %s",
+                exc_type.__name__, str(exc_val),
+                exc_info=(exc_type, exc_val, exc_tb)
+            )
 
 
 class MysqlUpload:
@@ -1624,3 +1974,46 @@ class OptimizeDatas:
 
 if __name__ == '__main__':
     pass
+
+    # 初始化上传器
+    uploader = MySQLUploader(
+        username='root',
+        password='1',
+        host='localhost',
+        port=3306,
+        enable_logging=True,
+        log_level='INFO'
+    )
+
+    # 定义列和数据类型
+    columns = {
+        'id': 'INT',
+        'name': 'VARCHAR(255)',
+        'age': 'INT',
+        'salary': 'DECIMAL(10,2)',
+        '日期': 'DATE'
+    }
+
+    # 准备数据
+    data = [
+        {'name': 'Alice', 'age': 30, 'salary': 50000.50, '日期': '2023-01-15'},
+        {'name': 'Bob', 'age': 25, 'salary': 45000.75, '日期': '2023-02-20'},
+        {'name': 'Charlie', 'age': 35, 'salary': 60000.00, '日期': '2023-01-10'}
+    ]
+
+    # 上传数据
+    uploader.upload_data(
+        db_name='test_db',
+        table_name='employees',
+        data=data,
+        columns=columns,
+        primary_keys=[],
+        check_duplicate=True,
+        replace=True,
+        duplicate_columns=['name'],
+        allow_null=False,
+        partition_by='month'  # 按月分表
+    )
+
+    # 关闭上传器
+    uploader.close()

{mdbq-3.9.0.dist-info → mdbq-3.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.0
+Version: 3.9.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.9.0.dist-info → mdbq-3.9.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=7hLUrBXQAGj08UbPoot9b_BwLXRkc-RH_nJSvG9AqTc,17
+mdbq/__version__.py,sha256=LC0UP2VyG12RJ8LWMMTlZDfZCrzzKDWmgNe41Gr89BE,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -8,7 +8,7 @@ mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=L_UR7TqcZoHZj6dWVZe-ai6X2yc_oULPyUzKy7DHbOw,74493
+mdbq/mysql/mysql.py,sha256=2xjf7j-6PSAmxdubYjwkh71n0Rhdum-VCkvTmQN-V3U,91100
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -22,7 +22,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
-mdbq-3.9.0.dist-info/METADATA,sha256=pd--meyNjH8KaX-ZgnSHqWN9GbEyx59Atb1Wgs6BqVc,363
-mdbq-3.9.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.9.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.9.0.dist-info/RECORD,,
+mdbq-3.9.1.dist-info/METADATA,sha256=2J1EsfDaQ5gl0dAjFvNDwF2U0xB1R9jmmICm9-XwH7s,363
+mdbq-3.9.1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.1.dist-info/RECORD,,