mdbq 3.8.19__py3-none-any.whl → 3.9.1__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.8.19'
+VERSION = '3.9.1'

mdbq/mysql/mysql.py

@@ -11,7 +11,8 @@ from sqlalchemy import create_engine
 import os
 import logging
 from mdbq.other import otk
-
+from dbutils.pooled_db import PooledDB
+from typing import Union, List, Dict, Optional, Any, Tuple
 warnings.filterwarnings('ignore')
 """
 建表流程:
@@ -43,6 +44,872 @@ def count_decimal_places(num_str):
     return 0, 0
 
 
+class MySQLUploader:
+    def __init__(
+            self,
+            username: str,
+            password: str,
+            host: str = 'localhost',
+            port: int = 3306,
+            charset: str = 'utf8mb4',
+            collation: str = 'utf8mb4_0900_ai_ci',
+            enable_logging: bool = False,
+            log_level: str = 'ERROR',
+            max_retries: int = 10,
+            retry_interval: int = 10,
+            pool_size: int = 5,
+            connect_timeout: int = 10,
+            read_timeout: int = 30,
+            write_timeout: int = 30,
+            ssl: Optional[Dict] = None
+    ):
+        """
+        初始化MySQL上传工具
+
+        :param username: 数据库用户名
+        :param password: 数据库密码
+        :param host: 数据库主机地址，默认为localhost
+        :param port: 数据库端口，默认为3306
+        :param charset: 字符集，默认为utf8mb4
+        :param collation: 排序规则，默认为utf8mb4_0900_ai_ci
+        :param enable_logging: 是否启用日志，默认为False
+        :param log_level: 日志级别，默认为ERROR
+        :param max_retries: 最大重试次数，默认为10
+        :param retry_interval: 重试间隔(秒)，默认为10
+        :param pool_size: 连接池大小，默认为5
+        :param connect_timeout: 连接超时(秒)，默认为10
+        :param read_timeout: 读取超时(秒)，默认为30
+        :param write_timeout: 写入超时(秒)，默认为30
+        :param ssl: SSL配置字典，默认为None
+        """
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+        self.charset = charset
+        self.collation = collation
+        self.max_retries = max(max_retries, 1)  # 至少重试1次
+        self.retry_interval = max(retry_interval, 1)  # 至少间隔1秒
+        self.pool_size = max(pool_size, 1)  # 至少1个连接
+        self.connect_timeout = connect_timeout
+        self.read_timeout = read_timeout
+        self.write_timeout = write_timeout
+        self.ssl = ssl
+        self._prepared_statements = {}  # 预处理语句缓存
+        self._max_cached_statements = 100  # 最大缓存语句数
+
+        # 初始化日志
+        if enable_logging:
+            self._init_logging(log_level)
+        else:
+            self.logger = None
+
+        # 创建连接池
+        self.pool = self._create_connection_pool()
+
+    def _init_logging(self, log_level: str):
+        """初始化日志配置"""
+        valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+        level = log_level.upper() if log_level.upper() in valid_levels else 'ERROR'
+
+        logging.basicConfig(
+            level=getattr(logging, level),
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[logging.StreamHandler()]
+        )
+        self.logger = logging.getLogger('MySQLUploader')
+
+    def _create_connection_pool(self) -> PooledDB:
+        """创建数据库连接池"""
+        pool_params = {
+            'creator': pymysql,
+            'host': self.host,
+            'port': self.port,
+            'user': self.username,
+            'password': self.password,
+            'charset': self.charset,
+            'cursorclass': pymysql.cursors.DictCursor,
+            'maxconnections': self.pool_size,
+            'ping': 7,  # 连接检查
+            'connect_timeout': self.connect_timeout,
+            'read_timeout': self.read_timeout,
+            'write_timeout': self.write_timeout,
+            'autocommit': False
+        }
+
+        if self.ssl:
+            required_keys = {'ca', 'cert', 'key'}
+            if not all(k in self.ssl for k in required_keys):
+                raise ValueError("SSL配置必须包含ca、cert和key")
+            pool_params['ssl'] = {
+                'ca': self.ssl['ca'],
+                'cert': self.ssl['cert'],
+                'key': self.ssl['key'],
+                'check_hostname': self.ssl.get('check_hostname', False)
+            }
+
+        try:
+            pool = PooledDB(**pool_params)
+            return pool
+        except Exception as e:
+            if self.logger:
+                self.logger.error("连接池创建失败: %s", str(e))
+            raise ConnectionError(f"连接池创建失败: {str(e)}")
+
+    def _validate_datetime(self, value):
+        formats = [
+            '%Y-%m-%d %H:%M:%S',
+            '%Y-%m-%d',
+            '%Y/%m/%d %H:%M:%S',
+            '%Y/%m/%d',
+            '%Y%m%d',
+            '%Y-%m-%dT%H:%M:%S',  # ISO格式
+            '%Y-%m-%d %H:%M:%S.%f'  # 带毫秒
+        ]
+        for fmt in formats:
+            try:
+                return datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
+            except ValueError:
+                continue
+        raise ValueError(f"无效的日期格式: {value}")
+
+    def _validate_identifier(self, identifier: str) -> str:
+        """
+        验证并清理数据库标识符(数据库名、表名、列名)
+        防止SQL注入和非法字符
+
+        :param identifier: 要验证的标识符
+        :return: 清理后的安全标识符
+        :raises ValueError: 如果标识符无效
+        """
+        if not identifier or not isinstance(identifier, str):
+            error_msg = f"无效的标识符: {identifier}"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # 移除可能有害的字符，只保留字母、数字、下划线和美元符号
+        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '', identifier)
+        if not cleaned:
+            error_msg = f"无法清理异常标识符: {identifier}"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # 检查是否为MySQL保留字
+        mysql_keywords = {
+            'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
+            'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
+        }
+        if cleaned.lower() in mysql_keywords:
+            if self.logger:
+                self.logger.warning("存在MySQL保留字: %s", cleaned)
+            return f"`{cleaned}`"
+
+        return cleaned
+
+    def _validate_value(self, value: Any, column_type: str) -> Any:
+        """
+        验证并清理数据值，根据列类型进行适当转换
+
+        :param value: 要验证的值
+        :param column_type: 列的数据类型
+        :return: 清理后的值
+        :raises ValueError: 如果值转换失败
+        """
+        if value is None:
+            return None
+
+        try:
+            column_type_lower = column_type.lower()
+
+            if 'int' in column_type_lower:
+                return int(value) if value is not None else None
+            elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
+                return float(value) if value is not None else None
+            elif '日期' in column_type_lower or 'time' in column_type_lower:
+                if isinstance(value, (datetime.datetime, pd.Timestamp)):
+                    return value.strftime('%Y-%m-%d %H:%M:%S')
+                elif isinstance(value, str):
+                    try:
+                        return self._validate_datetime(value)  # 使用专门的日期验证方法
+                    except ValueError as e:
+                        raise ValueError(f"无效日期格式: {value} - {str(e)}")
+                return str(value)
+            elif 'char' in column_type_lower or 'text' in column_type_lower:
+                # 防止SQL注入
+                if isinstance(value, str):
+                    return value.replace('\\', '\\\\').replace("'", "\\'")
+                return str(value)
+            elif 'json' in column_type_lower:
+                import json
+                return json.dumps(value) if value is not None else None
+            else:
+                return value
+        except (ValueError, TypeError) as e:
+            error_msg = f"数据类型转换异常 {value} to type {column_type}: {str(e)}"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+    def _execute_with_retry(self, func, *args, **kwargs):
+        """
+        带重试机制的SQL执行装饰器
+
+        :param func: 要执行的函数
+        :param args: 位置参数
+        :param kwargs: 关键字参数
+        :return: 函数执行结果
+        :raises Exception: 如果所有重试都失败
+        """
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(self.max_retries):
+                try:
+                    result = func(*args, **kwargs)
+                    if attempt > 0 and self.logger:
+                        self.logger.info("Operation succeeded after %d retries", attempt)
+                    return result
+                except (pymysql.OperationalError, pymysql.err.MySQLError) as e:
+                    last_exception = e
+                    if attempt < self.max_retries - 1:
+                        wait_time = self.retry_interval * (attempt + 1)
+                        if self.logger:
+                            self.logger.warning(
+                                "尝试 %d/%d 失败: %s. %d秒后重试...",
+                                attempt + 1, self.max_retries, str(e), wait_time
+                            )
+                        time.sleep(wait_time)
+                        # 尝试重新连接
+                        try:
+                            self.pool = self._create_connection_pool()
+                        except Exception as reconnect_error:
+                            if self.logger:
+                                self.logger.error("重连失败: %s", str(reconnect_error))
+                        continue
+                    else:
+                        if self.logger:
+                            self.logger.error(
+                                "Operation failed after %d attempts. Last error: %s",
+                                self.max_retries, str(e)
+                            )
+                except pymysql.IntegrityError as e:
+                    # 完整性错误通常不需要重试
+                    if self.logger:
+                        self.logger.error("完整性约束错误: %s", str(e))
+                    raise e
+                except Exception as e:
+                    last_exception = e
+                    if self.logger:
+                        self.logger.error("发生意外错误: %s", str(e))
+                    break
+
+            raise last_exception if last_exception else Exception("发生未知错误")
+
+        return wrapper(*args, **kwargs)
+
+    def _get_connection(self):
+        """从连接池获取连接"""
+        try:
+            conn = self.pool.connection()
+            if self.logger:
+                self.logger.debug("成功获取数据库连接")
+            return conn
+        except Exception as e:
+            if self.logger:
+                self.logger.error("连接数据库失败: %s", str(e))
+            raise ConnectionError(f"连接数据库失败: {str(e)}")
+
+    def _check_database_exists(self, db_name: str) -> bool:
+        """检查数据库是否存在"""
+        db_name = self._validate_identifier(db_name)
+        sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
+
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql, (db_name,))
+                    exists = bool(cursor.fetchone())
+                    if self.logger:
+                        self.logger.debug("数据库 %s 已存在: %s", db_name, exists)
+                    return exists
+        except Exception as e:
+            if self.logger:
+                self.logger.error("检查数据库是否存在时出错: %s", str(e))
+            raise
+
+    def _create_database(self, db_name: str):
+        """创建数据库"""
+        db_name = self._validate_identifier(db_name)
+        sql = f"CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}"
+
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                conn.commit()
+                if self.logger:
+                    self.logger.info("数据库 %s 创建成功", db_name)
+        except Exception as e:
+            if self.logger:
+                self.logger.error("无法创建数据库 %s: %s", db_name, str(e))
+            conn.rollback()
+            raise
+
+    def _check_table_exists(self, db_name: str, table_name: str) -> bool:
+        """检查表是否存在"""
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = """
+        SELECT TABLE_NAME 
+        FROM INFORMATION_SCHEMA.TABLES 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        """
+
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql, (db_name, table_name))
+                    exists = bool(cursor.fetchone())
+                    return exists
+        except Exception as e:
+            if self.logger:
+                self.logger.error("检查数据表是否存在时发生未知错误: %s", str(e))
+            raise
+
+    def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
+        """获取表的列名和数据类型"""
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = """
+        SELECT COLUMN_NAME, DATA_TYPE 
+        FROM INFORMATION_SCHEMA.COLUMNS 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        ORDER BY ORDINAL_POSITION
+        """
+
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql, (db_name, table_name))
+                    columns = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
+                    if self.logger:
+                        self.logger.debug("获取表 %s.%s 的列信息: %s", db_name, table_name, columns)
+                    return columns
+        except Exception as e:
+            if self.logger:
+                self.logger.error("无法获取表列信息: %s", str(e))
+            raise
+
+    def _prepare_data(
+            self,
+            data: Union[Dict, List[Dict], pd.DataFrame],
+            columns: Dict[str, str],
+            allow_null: bool = False
+    ) -> List[Dict]:
+        """
+        准备要上传的数据，验证并转换数据类型
+
+        :param data: 输入数据
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param allow_null: 是否允许空值
+        :return: 准备好的数据列表
+        :raises ValueError: 如果数据验证失败
+        """
+        # 统一数据格式为字典列表
+        if isinstance(data, pd.DataFrame):
+            try:
+                data = data.replace({pd.NA: None}).to_dict('records')
+            except Exception as e:
+                if self.logger:
+                    self.logger.error("Failed to convert DataFrame to dict: %s", str(e))
+                raise ValueError(f"Failed to convert DataFrame to dict: {str(e)}")
+        elif isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
+            error_msg = "Data must be a dict, list of dicts, or DataFrame"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        prepared_data = []
+        for row_idx, row in enumerate(data, 1):
+            prepared_row = {}
+            for col_name, col_type in columns.items():
+                # 跳过id列，不允许外部传入id
+                if col_name.lower() == 'id':
+                    continue
+
+                if col_name not in row:
+                    if not allow_null:
+                        error_msg = f"Row {row_idx}: Missing required column '{col_name}' in data"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise ValueError(error_msg)
+                    prepared_row[col_name] = None
+                else:
+                    try:
+                        prepared_row[col_name] = self._validate_value(row[col_name], col_type)
+                    except ValueError as e:
+                        error_msg = f"Row {row_idx}, column '{col_name}': {str(e)}"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise ValueError(error_msg)
+            prepared_data.append(prepared_row)
+
+        if self.logger:
+            self.logger.debug("已准备 %d 行数据", len(prepared_data))
+        return prepared_data
+
+    def _get_partition_table_name(self, table_name: str, date_value: str, partition_by: str) -> str:
+        """
+        获取分表名称
+
+        :param table_name: 基础表名
+        :param date_value: 日期值
+        :param partition_by: 分表方式 ('year' 或 'month')
+        :return: 分表名称
+        :raises ValueError: 如果日期格式无效或分表方式无效
+        """
+        try:
+            date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
+        except ValueError:
+            try:
+                date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
+            except ValueError:
+                error_msg = f"无效的日期格式: {date_value}"
+                if self.logger:
+                    self.logger.error("无效的日期格式: %s", date_value)
+                raise ValueError(error_msg)
+
+        if partition_by == 'year':
+            return f"{table_name}_{date_obj.year}"
+        elif partition_by == 'month':
+            return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
+        else:
+            error_msg = "partition_by must be 'year' or 'month'"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+    def _create_table(
+            self,
+            db_name: str,
+            table_name: str,
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]] = None,
+            date_column: Optional[str] = None,
+            indexes: Optional[List[str]] = None,
+            unique_columns: Optional[List[str]] = None
+    ):
+        """
+        创建数据表
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param date_column: 日期列名，如果存在将设置为索引
+        :param indexes: 需要创建索引的列列表
+        :param unique_columns: 需要创建唯一索引的列列表
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not columns:
+            error_msg = "No columns specified for table creation"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # 构建列定义SQL
+        column_defs = ["`id` INT NOT NULL AUTO_INCREMENT"]
+
+        # 添加其他列定义
+        for col_name, col_type in columns.items():
+            # 跳过id列，因为已经在前面添加了
+            if col_name.lower() == 'id':
+                continue
+            safe_col_name = self._validate_identifier(col_name)
+            col_def = f"`{safe_col_name}` {col_type}"
+
+            # 添加NOT NULL约束
+            if not col_type.lower().startswith('json'):
+                col_def += " NOT NULL"
+
+            column_defs.append(col_def)
+
+        # 添加主键定义
+        if primary_keys:
+            # 确保id在主键中
+            if 'id' not in [pk.lower() for pk in primary_keys]:
+                primary_keys = ['id'] + primary_keys
+        else:
+            # 如果没有指定主键，则使用id作为主键
+            primary_keys = ['id']
+
+        # 添加主键定义
+        safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
+        primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+
+        # 添加唯一索引定义
+        unique_index_sql = ""
+        if unique_columns:
+            for col in unique_columns:
+                if col.lower() != 'id' and col in columns:
+                    safe_col = self._validate_identifier(col)
+                    unique_index_sql += f", UNIQUE KEY `uk_{safe_col}` (`{safe_col}`)"
+
+        # 构建完整SQL
+        sql = f"""
+        CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
+            {','.join(column_defs)}
+            {primary_key_sql}
+            {unique_index_sql}
+        ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
+        """
+
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                    if self.logger:
+                        self.logger.info("表 %s.%s 创建成功", db_name, table_name)
+
+                # 添加普通索引
+                index_statements = []
+
+                # 日期列索引
+                if date_column and date_column in columns:
+                    safe_date_col = self._validate_identifier(date_column)
+                    index_statements.append(
+                        f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
+                    )
+
+                # 其他索引
+                if indexes:
+                    for idx_col in indexes:
+                        if idx_col in columns:
+                            safe_idx_col = self._validate_identifier(idx_col)
+                            index_statements.append(
+                                f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)"
+                            )
+
+                # 执行所有索引创建语句
+                if index_statements:
+                    with conn.cursor() as cursor:
+                        for stmt in index_statements:
+                            cursor.execute(stmt)
+                            if self.logger:
+                                self.logger.debug("Executed index statement: %s", stmt)
+
+                conn.commit()
+                if self.logger:
+                    self.logger.info("All indexes created successfully for %s.%s", db_name, table_name)
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error("创建表 %s.%s 失败: %s", db_name, table_name, str(e))
+            conn.rollback()
+            raise
+
+    def upload_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: Union[Dict, List[Dict], pd.DataFrame],
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]] = None,
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            allow_null: bool = False,
+            partition_by: Optional[str] = None,
+            partition_date_column: str = '日期',
+            auto_create: bool = True,
+            replace: bool = False,
+            indexes: Optional[List[str]] = None
+    ):
+        """
+        上传数据到数据库
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要上传的数据
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param check_duplicate: 是否检查重复，默认为False
+        :param duplicate_columns: 用于检查重复的列列表，如果不指定则使用所有列
+        :param allow_null: 是否允许空值，默认为False
+        :param partition_by: 分表方式 ('year' 或 'month')，默认为None不分表
+        :param partition_date_column: 用于分表的日期列名，默认为'date'
+        :param auto_create: 是否自动创建不存在的数据库或表，默认为True
+        :param replace: 是否使用REPLACE代替INSERT，默认为False
+        :param indexes: 需要创建索引的列列表
+        :raises ValueError: 如果参数无效或操作失败
+        """
+        if self.logger:
+            self.logger.info(
+                "开始上传数据到 %s.%s (分表方式=%s, 替换模式=%s)",
+                db_name, table_name, partition_by, replace
+            )
+
+        # 验证参数
+        if not columns:
+            error_msg = "Columns specification is required"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        if partition_by and partition_by not in ['year', 'month']:
+            error_msg = "分表方式必须是 'year' 或 'month'"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # 准备数据
+        prepared_data = self._prepare_data(data, columns, allow_null)
+
+        # 检查数据库是否存在
+        if not self._check_database_exists(db_name):
+            if auto_create:
+                self._create_database(db_name)
+            else:
+                error_msg = f"Database '{db_name}' does not exist"
+                if self.logger:
+                    self.logger.error(error_msg)
+                raise ValueError(error_msg)
+
+        # 确定唯一索引列
+        unique_columns = None
+        if check_duplicate:
+            unique_columns = duplicate_columns if duplicate_columns else [col for col in columns.keys() if
+                                                                          col.lower() != 'id']
+
+        # 处理分表逻辑
+        if partition_by:
+            # 分组数据按分表
+            partitioned_data = {}
+            for row in prepared_data:
+                if partition_date_column not in row:
+                    error_msg = f"异常缺失列 '{partition_date_column}'"
+                    if self.logger:
+                        self.logger.error(error_msg)
+                    raise ValueError(error_msg)
+                part_table = self._get_partition_table_name(table_name, str(row[partition_date_column]), partition_by)
+                if part_table not in partitioned_data:
+                    partitioned_data[part_table] = []
+                partitioned_data[part_table].append(row)
+
+            # 对每个分表执行上传
+            for part_table, part_data in partitioned_data.items():
+                self._upload_to_table(
+                    db_name, part_table, part_data, columns,
+                    primary_keys, check_duplicate, duplicate_columns,
+                    allow_null, auto_create, partition_date_column,
+                    replace, indexes, unique_columns
+                )
+        else:
+            # 不分表，直接上传
+            self._upload_to_table(
+                db_name, table_name, prepared_data, columns,
+                primary_keys, check_duplicate, duplicate_columns,
+                allow_null, auto_create, partition_date_column,
+                replace, indexes, unique_columns
+            )
+
+        if self.logger:
+            self.logger.info(
+                "成功上传 %d 行数据到 %s.%s",
+                len(prepared_data), db_name, table_name
+            )
+
+    def _upload_to_table(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]],
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            allow_null: bool,
+            auto_create: bool,
+            date_column: Optional[str],
+            replace: bool,
+            indexes: Optional[List[str]],
+            unique_columns: Optional[List[str]] = None
+    ):
+        """实际执行表上传的内部方法"""
+        # 检查表是否存在
+        if not self._check_table_exists(db_name, table_name):
+            if auto_create:
+                self._create_table(db_name, table_name, columns, primary_keys, date_column, indexes, unique_columns)
+            else:
+                error_msg = f"Table '{db_name}.{table_name}' does not exist"
+                if self.logger:
+                    self.logger.error(error_msg)
+                raise ValueError(error_msg)
+
+        # 获取表结构并验证
+        table_columns = self._get_table_columns(db_name, table_name)
+        if not table_columns:
+            error_msg = f"Failed to get columns for table '{db_name}.{table_name}'"
+            if self.logger:
+                self.logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # 验证数据列与表列匹配
+        for col in columns:
+            if col not in table_columns:
+                error_msg = f"Column '{col}' not found in table '{db_name}.{table_name}'"
+                if self.logger:
+                    self.logger.error(error_msg)
+                raise ValueError(error_msg)
+
+        # 插入数据
+        self._insert_data(
+            db_name, table_name, data, columns,
+            check_duplicate, duplicate_columns,
+            replace=replace
+        )
+
+    def _insert_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            batch_size: int = 1000,
+            replace: bool = False
+    ):
+        """
+        插入数据到表中
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要插入的数据
+        :param columns: 列名和数据类型字典
+        :param check_duplicate: 是否检查重复
+        :param duplicate_columns: 用于检查重复的列列表
+        :param batch_size: 批量插入的大小
+        :param replace: 是否使用REPLACE代替INSERT
+        :raises Exception: 如果插入失败
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not data:
+            if self.logger:
+                self.logger.warning("No data to insert into %s.%s", db_name, table_name)
+            return
+
+        # 获取所有列名
+        all_columns = [col for col in columns.keys() if col.lower() != 'id']
+        safe_columns = [self._validate_identifier(col) for col in all_columns]
+        placeholders = ','.join(['%s'] * len(safe_columns))
+
+        # 构建SQL语句
+        operation = "REPLACE" if replace else "INSERT IGNORE" if check_duplicate else "INSERT"
+
+        if check_duplicate and not replace:
+            # 当check_duplicate=True时，使用INSERT IGNORE来跳过重复记录
+            sql = f"""
+            {operation} INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            """
+        else:
+            sql = f"""
+            {operation} INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            """
+
+        if len(self._prepared_statements) >= self._max_cached_statements:
+            # 移除最旧的缓存
+            oldest_key = next(iter(self._prepared_statements))
+            del self._prepared_statements[oldest_key]
+
+        # 缓存预处理语句
+        cache_key = f"{db_name}.{table_name}.{operation}.{check_duplicate}"
+        if cache_key not in self._prepared_statements:
+            self._prepared_statements[cache_key] = sql
+            if self.logger:
+                self.logger.debug("已缓存预处理语句: %s", cache_key)
+
+        # 分批插入数据
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                for i in range(0, len(data), batch_size):
+                    batch = data[i:i + batch_size]
+                    # 准备批量数据
+                    values = []
+                    for row in batch:
+                        row_values = []
+                        for col in all_columns:
+                            row_values.append(row.get(col))
+                        values.append(row_values)
+
+                    # 执行批量插入
+                    try:
+                        start_time = time.time()
+                        cursor.executemany(sql, values)
+                        conn.commit()  # 每个批次提交一次
+                        if self.logger:
+                            self.logger.debug(
+                                "成功插入批次 %d-%d/%d 到 %s.%s, 耗时 %.2f 秒",
+                                i + 1, min(i + batch_size, len(data)), len(data),
+                                db_name, table_name, time.time() - start_time
+                            )
+                    except Exception as e:
+                        conn.rollback()
+                        error_msg = f"Failed to insert batch {i + 1}-{min(i + batch_size, len(data))}/{len(data)} into {db_name}.{table_name}: {str(e)}"
+                        if self.logger:
+                            self.logger.error(error_msg)
+                        raise Exception(error_msg)
+
+    def close(self):
+        """关闭连接池"""
+        if hasattr(self, 'pool') and self.pool:
+            try:
+                # 先关闭所有连接
+                while True:
+                    conn = getattr(self.pool, '_connections', None)
+                    if not conn or not conn.queue:
+                        break
+                    try:
+                        conn = self.pool.connection()
+                        conn.close()
+                    except:
+                        pass
+
+                # 然后关闭连接池
+                self.pool.close()
+                if self.logger:
+                    self.logger.info("连接池已成功关闭")
+            except Exception as e:
+                if self.logger:
+                    self.logger.error("关闭连接池失败: %s", str(e))
+                raise
+        self.pool = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+        if exc_type is not None and self.logger:
+            self.logger.error(
+                "Exception occurred: %s: %s",
+                exc_type.__name__, str(exc_val),
+                exc_info=(exc_type, exc_val, exc_tb)
+            )
+
+
 class MysqlUpload:
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
         self.username = username
@@ -1107,3 +1974,46 @@ class OptimizeDatas:
 
 if __name__ == '__main__':
     pass
+
+    # 初始化上传器
+    uploader = MySQLUploader(
+        username='root',
+        password='1',
+        host='localhost',
+        port=3306,
+        enable_logging=True,
+        log_level='INFO'
+    )
+
+    # 定义列和数据类型
+    columns = {
+        'id': 'INT',
+        'name': 'VARCHAR(255)',
+        'age': 'INT',
+        'salary': 'DECIMAL(10,2)',
+        '日期': 'DATE'
+    }
+
+    # 准备数据
+    data = [
+        {'name': 'Alice', 'age': 30, 'salary': 50000.50, '日期': '2023-01-15'},
+        {'name': 'Bob', 'age': 25, 'salary': 45000.75, '日期': '2023-02-20'},
+        {'name': 'Charlie', 'age': 35, 'salary': 60000.00, '日期': '2023-01-10'}
+    ]
+
+    # 上传数据
+    uploader.upload_data(
+        db_name='test_db',
+        table_name='employees',
+        data=data,
+        columns=columns,
+        primary_keys=[],
+        check_duplicate=True,
+        replace=True,
+        duplicate_columns=['name'],
+        allow_null=False,
+        partition_by='month'  # 按月分表
+    )
+
+    # 关闭上传器
+    uploader.close()

mdbq/spider/aikucun.py

@@ -257,13 +257,13 @@ class AikuCun:
             )
             print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
             # print(res.json())
-            if not res.json()['success']:
+            if not res.json().get('success', None):
                 print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
                 continue
-            if not res.json()['data']['rows']:
+            if not res.json().get('data', {}).get('rows', None):
                 print("返回的数据字典异常, ['data']['rows'] 不能为空")
                 num += 1
                 self.error_count += 1
@@ -479,7 +479,10 @@ def main(start_date, end_date=None, item_type=['spu']):
 
 if __name__ == '__main__':
     main(
-        start_date='2025-03-25',
-        # end_date='2025-03-26',  # 不传则默认到今天
-        item_type=['spu', 'sku']
+        start_date='2025-05-13',
+        # end_date='2025-04-28',  # 不传则默认到今天
+        item_type=[
+            # 'spu',
+            'sku'
+        ]
     )

{mdbq-3.8.19.dist-info → mdbq-3.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.19
+Version: 3.9.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.19.dist-info → mdbq-3.9.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=adbn7k6HeaPn3waOOAb2wMca7SSJxTRZ8MEbtLkPAeA,18
+mdbq/__version__.py,sha256=LC0UP2VyG12RJ8LWMMTlZDfZCrzzKDWmgNe41Gr89BE,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -8,7 +8,7 @@ mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=umcLpw5cYGNNJnEjBLh_bgBXeh5LntPKFm8VslQ01ow,55030
+mdbq/mysql/mysql.py,sha256=2xjf7j-6PSAmxdubYjwkh71n0Rhdum-VCkvTmQN-V3U,91100
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -21,8 +21,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=QfyUtXMuPZ5mJVNDUlFa_ltFXiCCTccBz6MT3YT-7HI,19742
-mdbq-3.8.19.dist-info/METADATA,sha256=yheVi8Il8o8Py_HUlKc8s-gCako_d8RktLqdNa-TPSA,364
-mdbq-3.8.19.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.19.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.19.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
+mdbq-3.9.1.dist-info/METADATA,sha256=2J1EsfDaQ5gl0dAjFvNDwF2U0xB1R9jmmICm9-XwH7s,363
+mdbq-3.9.1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.1.dist-info/RECORD,,