mdbq 3.8.18__py3-none-any.whl → 3.9.0__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.8.18'
+VERSION = '3.9.0'

mdbq/aggregation/optimize.py

@@ -0,0 +1,474 @@
+# -*- coding:utf-8 -*-
+import pymysql
+import logging
+from typing import List, Optional, Dict
+import time
+import re
+import os
+import hashlib
+from dbutils.pooled_db import PooledDB
+from mdbq.log import spider_logging
+from mdbq.config import config
+import threading
+import queue
+
+dir_path = os.path.expanduser("~")
+config_file = os.path.join(dir_path, 'spd.txt')
+my_cont = config.read_config(config_file)
+username, password, port = my_cont['username'], my_cont['password'], my_cont['port']
+host = '127.0.0.1'
+logger = spider_logging.setup_logging(reMoveOldHandler=True, filename='optimize.log')
+
+
+class MySQLDeduplicator:
+
+    def __init__(self, host: str, username: str, password: str, port: int = 3306):
+        self.pool = PooledDB(
+            creator=pymysql,
+            maxconnections=10,  # 最大连接数
+            mincached=2,  # 初始化空闲连接数
+            maxcached=5,  # 空闲连接最大缓存数
+            blocking=True,
+            host=host,
+            port=int(port),
+            user=username,
+            password=password,
+            ping=1,
+            charset='utf8mb4',
+            cursorclass=pymysql.cursors.DictCursor
+        )
+        self.set_typ = {
+            '日期': 'date',
+            '更新时间': 'timestamp',
+        }
+        self.tables_to_reset = queue.Queue()  # 线程安全队列
+        self.delay_time = 120  # 延迟重置自增 id
+        self.lock = threading.Lock()  # 用于关键操作同步
+
+    def get_table_in_databases(self, db_list=None, reset_id=False):
+        """
+        reset_id: 是否重置自增 id
+        """
+        if not db_list:
+            return
+        connection = self.get_connection()
+        res = []
+        for db_name in db_list:
+            try:
+                with connection.cursor() as cursor:
+                    cursor.execute(f"USE `{db_name}`")
+                    cursor.execute("SHOW TABLES")
+                    tables = cursor.fetchall()
+                    for index, item in enumerate(tables):
+                        res.append(
+                            {
+                                'db_name': db_name,
+                                'table_name': item.get(f'Tables_in_{db_name}', ''),
+                                'reset_id': reset_id,
+                            }
+                        )
+            except:
+                pass
+        connection.close()
+        return res
+
+    def deduplicate(
+            self,
+            tables_list: List[Dict],
+            order_column: str = "更新时间",
+            order_direction: str = "DESC",
+            batch_size: int = 10000,
+            id_column: str = "id",
+            recent_months: Optional[int] = None
+    ) -> bool:
+        """
+        执行多表去重操作
+        :param tables_list: 目标表配置列表，每个元素为字典，包含db_name, table_name, unique_keys（可选）, reset_id（可选）
+        :param order_column: 排序字段
+        :param order_direction: 排序方向 (ASC/DESC)
+        :param batch_size: 批量删除批次大小
+        :param id_column: 自增列名称
+        :return: 是否全部成功
+        """
+        if recent_months is not None and (not isinstance(recent_months, int) or recent_months < 1):
+            logger.error("recent_months必须为None或正整数")
+            return False
+        for table_config in tables_list:
+            config = {
+                'order_column': order_column,
+                'order_direction': order_direction,
+                'batch_size': batch_size,
+                'id_column': id_column,
+                'reset_id': table_config.get('reset_id', False),  # 处理默认值
+                'unique_keys': table_config.get('unique_keys', None),
+                'recent_months': recent_months,
+            }
+            config.update(table_config)
+            self._deduplicate_single_table(**config)
+
+    def _deduplicate_single_table(
+            self,
+            db_name: str,
+            table_name: str,
+            unique_keys: Optional[List[str]],
+            order_column: str,
+            order_direction: str,
+            batch_size: int,
+            reset_id: bool,
+            id_column: str,
+            recent_months: Optional[int] = None
+    ):
+        """单表去重逻辑"""
+
+        # 获取数据库连接并检查有效性
+        connection = self.get_connection(db_name=db_name)
+        if not connection:
+            logger.error(f"连接数据库失败: {db_name}")
+            return False
+
+        temp_suffix = hashlib.md5(f"{table_name}{time.time()}".encode()).hexdigest()[:8]
+        temp_table = f"temp_{temp_suffix}"
+
+        try:
+            # 版本检查在check_db内部
+            if not self.check_db(db_name, table_name):
+                return False
+
+            with connection.cursor() as cursor:
+                # 主键重复检查
+                try:
+                    cursor.execute(f"""
+                        SELECT COUNT(*) AS total, 
+                               COUNT(DISTINCT `{id_column}`) AS distinct_count 
+                        FROM `{table_name}`
+                    """)
+                except pymysql.err.InternalError as e:
+                    if e.args[0] == pymysql.constants.ER.BAD_FIELD_ERROR:
+                        logger.warning(f"{db_name}/{table_name} 跳过主键检查（无{id_column}列）")
+                    else:
+                        raise
+                else:
+                    res = cursor.fetchone()
+                    if res['total'] != res['distinct_count']:
+                        logger.error(f"{db_name}/{table_name} 主键重复: {id_column}")
+                        return False
+
+                all_columns = self._get_table_columns(db_name, table_name)
+                # 自动生成unique_keys逻辑
+                if not unique_keys:
+                    exclude_set = {id_column.lower(), order_column.lower()}
+
+                    if not all_columns:
+                        logger.error(f"{db_name}/{table_name} 无法获取表列信息")
+                        return False
+
+                    # 排除id_column和order_column
+                    unique_keys = [
+                        col for col in all_columns
+                        if col.lower() not in exclude_set
+                           and col != id_column  # 额外确保大小写兼容
+                           and col != order_column
+                    ]
+                    # 检查剩余列是否有效
+                    if not unique_keys:
+                        unique_keys = all_columns
+                        logger.warning(f"{db_name}/{table_name} 使用全列作为唯一键: {all_columns}")
+                        return False
+                    # logger.info(f"自动生成unique_keys: {unique_keys}")
+                else:
+                    if not self._validate_columns(db_name, table_name, unique_keys):
+                        logger.error(f"{db_name}/{table_name} unique_keys中存在无效列名")
+                        return False
+
+                # 动态生成临时表名
+                partition_clause = ', '.join([f'`{col}`' for col in unique_keys])
+
+                # 使用参数化查询创建临时表
+                if self._validate_columns(db_name, table_name, [order_column]):
+                    order_clause = f"ORDER BY `{order_column}` {order_direction}" if order_column else ""
+                else:
+                    order_clause = ''
+
+                # 时间过滤
+                where_clause = ""
+                query_params = []
+                date_column_exists = '日期' in all_columns
+                if recent_months and recent_months > 0 and date_column_exists:
+                    where_clause = "WHERE `日期` >= DATE_SUB(CURDATE(), INTERVAL %s MONTH)"
+                    query_params.append(recent_months)
+                elif recent_months and not date_column_exists:
+                    logger.warning(f"{db_name}/{table_name} 忽略recent_months参数（无日期列）")
+
+                create_temp_sql = f"""
+                            CREATE TEMPORARY TABLE `{temp_table}` AS
+                            SELECT tmp_id FROM (
+                                SELECT `{id_column}` AS tmp_id,
+                                ROW_NUMBER() OVER (
+                                    PARTITION BY {partition_clause or '1'}
+                                    {order_clause}
+                                ) AS row_num
+                                FROM `{table_name}`
+                                {where_clause}
+                            ) t WHERE row_num > 1;
+                        """
+                cursor.execute(create_temp_sql, query_params)
+
+                logger.info(f'{db_name}/{table_name}  执行排重任务')
+                # 批量删除优化
+                iteration = 0
+                total_deleted = 0
+                while True and iteration < 10000:
+                    iteration += 1
+                    # 获取并删除临时表中的数据，避免重复处理
+                    cursor.execute(f"""
+                            SELECT tmp_id 
+                            FROM `{temp_table}` 
+                            LIMIT %s 
+                            FOR UPDATE;
+                        """, (batch_size,))
+                    batch = cursor.fetchall()
+                    if not batch:
+                        break
+                    ids = [str(row['tmp_id']) for row in batch]
+                    placeholder = ','.join(['%s'] * len(ids))
+
+                    if ids:
+                        try:
+                            # 删除主表数据
+                            cursor.execute(f"DELETE FROM `{table_name}` WHERE `{id_column}` IN ({placeholder})", ids)
+
+                            # 删除临时表中已处理的记录
+                            cursor.execute(f"DELETE FROM `{temp_table}` WHERE tmp_id IN ({placeholder})", ids)
+                        except pymysql.err.InternalError as e:
+                            if e.args[0] == pymysql.constants.ER.BAD_FIELD_ERROR:
+                                logger.error(f"{db_name}/{table_name}  无法通过 {id_column} 删除记录，请检查列存在性")
+                                return False
+                            raise
+
+                    total_deleted += cursor.rowcount
+                    connection.commit()
+                    logger.info(f"{db_name}/{table_name}  执行去重, 删除记录数: {total_deleted}")
+
+                if total_deleted > 0:
+                    logger.info(f"{db_name}/{table_name}  删除记录数总计: {total_deleted}")
+
+                # 线程安全操作队列
+                if reset_id:
+                    if not self._validate_columns(db_name, table_name, [id_column]):
+                        return True
+
+                    with self.lock:
+                        self.tables_to_reset.put((db_name, table_name, id_column))
+                    logger.info(f"{db_name}/{table_name} -> {self.delay_time}秒后重置自增id")
+                    threading.Timer(self.delay_time, self.delayed_reset_auto_increment).start()
+
+                return True
+        except Exception as e:
+            logger.error(f"{db_name}/{table_name} 去重操作异常: {e}", exc_info=True)
+            connection.rollback()
+            return False
+        finally:
+            with connection.cursor() as cursor:
+                cursor.execute(f"DROP TEMPORARY TABLE IF EXISTS `{temp_table}`")
+            connection.close()
+
+    def _get_table_columns(self, db_name: str, table_name: str) -> List[str]:
+        """获取表的列"""
+        try:
+            connection = self.get_connection(db_name=db_name)
+            with connection.cursor() as cursor:
+                cursor.execute(f"SHOW COLUMNS FROM `{table_name}`")
+                return [row["Field"] for row in cursor.fetchall()]
+        except pymysql.Error as e:
+            logging.error(f"{db_name}/{table_name} 获取列失败: {e}")
+            return []
+
+    def check_db(self, db_name: str, table_name: str) -> bool:
+        """数据库检查"""
+        try:
+            with self.get_connection() as conn:
+                with conn.cursor() as cursor:
+                    # 获取MySQL版本
+                    version = self._check_mysql_version(cursor)
+                    collation = 'utf8mb4_0900_ai_ci' if version >= 8.0 else 'utf8mb4_general_ci'
+
+                    # 创建数据库
+                    cursor.execute(f"""
+                        CREATE DATABASE IF NOT EXISTS `{db_name}` 
+                        CHARACTER SET utf8mb4 COLLATE {collation}
+                    """)
+                    conn.commit()
+
+                    # 切换数据库
+                    cursor.execute(f"USE `{db_name}`")
+
+                    # 检查表是否存在
+                    if not self._table_exists(cursor, table_name):
+                        self._create_table(cursor, table_name)
+                        conn.commit()
+                    return True
+        except Exception as e:
+            logger.error(f"{db_name}/{table_name} 数据库检查失败: {e}")
+            return False
+
+    def get_connection(self, db_name=None):
+        """从连接池获取连接"""
+        for _ in range(10):
+            try:
+                if db_name:
+                    connection = self.pool.connection()
+                    with connection.cursor() as cursor:
+                        cursor.execute(f'use {db_name};')
+                        return connection
+
+                return self.pool.connection()
+            except pymysql.Error as e:
+                logger.error(f"{db_name}  获取连接失败: {e}, 30秒后重试...")
+                time.sleep(30)
+        logger.error(f"{host}: {port}  数据库连接失败，已达最大重试次数")
+        return None
+
+    def _validate_identifier(self, name: str) -> bool:
+        """更严格的对象名验证（符合MySQL规范）"""
+        return re.match(r'^[\w$]+$', name) and len(name) <= 64
+
+    def _validate_columns(self, db_name: str, table_name: str, columns: List[str]) -> bool:
+        """验证列是否存在"""
+        if not all(self._validate_identifier(col) for col in columns):
+            return False
+        try:
+            connection = self.get_connection(db_name=db_name)
+            with connection.cursor() as cursor:
+                cursor.execute(f"SHOW COLUMNS FROM `{table_name}`")
+                existing_columns = {col['Field'] for col in cursor.fetchall()}
+                return all(col in existing_columns for col in columns)
+        except pymysql.Error as e:
+            logging.error(f"{db_name}/{table_name} 列验证失败: {e}")
+            return False
+
+    def _check_mysql_version(self, cursor) -> float:
+        """通过传入游标检查版本"""
+        cursor.execute("SELECT VERSION()")
+        return float(cursor.fetchone()['VERSION()'][:3])
+
+    def _table_exists(self, cursor, table_name: str) -> bool:
+        cursor.execute("SHOW TABLES LIKE %s", (table_name,))
+        return cursor.fetchone() is not None
+
+    def _create_table(self, cursor, table_name: str):
+        """安全建表逻辑"""
+        columns = ["`id` INT AUTO_INCREMENT PRIMARY KEY"]
+        for cn, ct in self.set_typ.items():
+            col_def = f"`{cn}` {ct.upper()} NOT NULL DEFAULT "
+            if 'INT' in ct:
+                col_def += '0'
+            elif 'TIMESTAMP' in ct:
+                col_def += 'CURRENT_TIMESTAMP'
+            else:
+                col_def += "''"
+            columns.append(col_def)
+        cursor.execute(f"""
+            CREATE TABLE `{table_name}` (
+                {', '.join(columns)}
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+        """)
+
+    def delayed_reset_auto_increment(self):
+        """线程安全的自增ID重置"""
+        while not self.tables_to_reset.empty():
+            try:
+                item = self.tables_to_reset.get_nowait()
+                self._safe_reset_auto_increment(*item)
+            except queue.Empty:
+                break
+
+    def _safe_reset_auto_increment(self, db_name: str, table_name: str, id_column: str):
+        """安全重置自增ID"""
+        with self.get_connection(db_name) as conn:
+            try:
+                with conn.cursor() as cursor:
+                    cursor.execute("START TRANSACTION")
+                    temp_table = f"reset_{hashlib.md5(table_name.encode()).hexdigest()[:8]}"
+                    backup_table = f"{table_name}_backup_{int(time.time())}"
+                    cursor.execute(f"CREATE TABLE `{temp_table}` LIKE `{table_name}`")
+                    cursor.execute(f"ALTER TABLE `{temp_table}` MODIFY COLUMN `{id_column}` INT NOT NULL")
+                    columns = self._get_table_columns(db_name, table_name)
+                    if id_column not in columns:
+                        logger.error(f"列 {id_column} 不存在于表 {table_name}")
+                        return False
+                    columns.remove(id_column)
+                    columns_str = ', '.join([f'`{col}`' for col in columns])
+                    insert_sql = f"""
+                        INSERT INTO `{temp_table}` (`{id_column}`, {columns_str})
+                        SELECT ROW_NUMBER() OVER (ORDER BY `{id_column}`), {columns_str}
+                        FROM `{table_name}` ORDER BY `{id_column}`
+                    """
+                    cursor.execute(insert_sql)
+                    cursor.execute(f"RENAME TABLE `{table_name}` TO `{backup_table}`, `{temp_table}` TO `{table_name}`")
+                    cursor.execute(f"ALTER TABLE `{table_name}` MODIFY COLUMN `{id_column}` INT AUTO_INCREMENT")
+                    cursor.execute(f"SELECT MAX(`{id_column}`) + 1 AS next_id FROM `{table_name}`")
+                    next_id = cursor.fetchone()['next_id'] or 1
+                    cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = {next_id}")
+                    cursor.execute(f"DROP TABLE IF EXISTS `{backup_table}`")
+                    cursor.execute(f"DROP TEMPORARY TABLE IF EXISTS `{temp_table}`")
+                    cursor.execute("COMMIT")
+                    logger.info(f'{db_name}/{table_name} 已重置自增id')
+            except Exception as e:
+                logger.error(f"{db_name}/{table_name} 重置自增id失败: {e}")
+                cursor.execute("ROLLBACK")
+                return False
+            finally:
+                conn.close()
+
+
+def main():
+    op = MySQLDeduplicator(
+        host=host,
+        username=username,
+        password=password,
+        port=port
+    )
+    op.delay_time = 600
+    # tables_list = [
+    #     {
+    #         'db_name': "测试库",
+    #         'table_name': "测试库2",
+    #         'reset_id': True,  # 可选, 默认 False
+    #         # 'unique_keys': ["日期", "店铺名称", "商品id"]
+    #     }
+    # ]
+    db_list = [
+        "京东数据3",
+        "属性设置3",
+        "推广数据2",
+        "推广数据_圣积天猫店",
+        "推广数据_淘宝店",
+        "推广数据_奥莱店",
+        "爱库存2",
+        "生意参谋3",
+        "生意经3",
+        "达摩盘3",
+        '人群画像2',
+        '商品人群画像2',
+        '市场数据3',
+        # '数据银行2'
+        # '回传数据',
+        # '大模型库',
+        '安全组',
+        # '视频数据',
+        # '聚合数据',
+    ]
+    tables_list = op.get_table_in_databases(db_list=db_list, reset_id=False)
+    op.deduplicate(
+        order_column = "更新时间",
+        order_direction = "DESC",
+        batch_size = 1000,
+        id_column = "id",
+        tables_list=tables_list,
+        recent_months=3,
+    )
+    logger.info(f'全部任务完成')
+
+
+if __name__ == "__main__":
+    main()

mdbq/mysql/mysql.py

@@ -12,6 +12,9 @@ import os
 import logging
 from mdbq.other import otk
 
+from dbutils.pooled_db import PooledDB
+from typing import Union, List, Dict, Optional, Any, Tuple
+
 warnings.filterwarnings('ignore')
 """
 建表流程:
@@ -43,6 +46,520 @@ def count_decimal_places(num_str):
     return 0, 0
 
 
+class MySQLUploader:
+    def __init__(
+            self,
+            username: str,
+            password: str,
+            host: str = 'localhost',
+            port: int = 3306,
+            charset: str = 'utf8mb4',
+            collation: str = 'utf8mb4_0900_ai_ci',
+            enable_logging: bool = False,
+            log_level: str = 'ERROR',
+            max_retries: int = 10,
+            retry_interval: int = 10,
+            pool_size: int = 5
+    ):
+        """
+        初始化MySQL上传工具
+
+        :param username: 数据库用户名
+        :param password: 数据库密码
+        :param host: 数据库主机地址，默认为localhost
+        :param port: 数据库端口，默认为3306
+        :param charset: 字符集，默认为utf8mb4
+        :param collation: 排序规则，默认为utf8mb4_0900_ai_ci
+        :param enable_logging: 是否启用日志，默认为False
+        :param log_level: 日志级别，默认为ERROR
+        :param max_retries: 最大重试次数，默认为10
+        :param retry_interval: 重试间隔(秒)，默认为10
+        :param pool_size: 连接池大小，默认为5
+        """
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+        self.charset = charset
+        self.collation = collation
+        self.max_retries = max_retries
+        self.retry_interval = retry_interval
+        self.pool_size = pool_size
+
+        # 初始化日志
+        if enable_logging:
+            self._init_logging(log_level)
+
+        # 创建连接池
+        self.pool = self._create_connection_pool()
+
+    def _init_logging(self, log_level: str):
+        """初始化日志配置"""
+        logging.basicConfig(
+            level=getattr(logging, log_level.upper(), logging.ERROR),
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger('MySQLUploader')
+
+    def _create_connection_pool(self):
+        """创建数据库连接池"""
+        return PooledDB(
+            creator=pymysql,
+            host=self.host,
+            port=self.port,
+            user=self.username,
+            password=self.password,
+            charset=self.charset,
+            maxconnections=self.pool_size,
+            cursorclass=pymysql.cursors.DictCursor
+        )
+
+    def _validate_identifier(self, identifier: str) -> str:
+        """
+        验证并清理数据库标识符(数据库名、表名、列名)
+        防止SQL注入和非法字符
+
+        :param identifier: 要验证的标识符
+        :return: 清理后的安全标识符
+        """
+        if not identifier or not isinstance(identifier, str):
+            raise ValueError(f"Invalid identifier: {identifier}")
+
+        # 移除可能有害的字符
+        cleaned = re.sub(r'[^a-zA-Z0-9_$]', '', identifier)
+        if not cleaned:
+            raise ValueError(f"Invalid identifier after cleaning: {identifier}")
+
+        return cleaned
+
+    def _validate_value(self, value: Any, column_type: str) -> Any:
+        """
+        验证并清理数据值，根据列类型进行适当转换
+
+        :param value: 要验证的值
+        :param column_type: 列的数据类型
+        :return: 清理后的值
+        """
+        if value is None:
+            return None
+
+        try:
+            if 'int' in column_type.lower():
+                return int(value) if value is not None else None
+            elif 'float' in column_type.lower() or 'double' in column_type.lower() or 'decimal' in column_type.lower():
+                return float(value) if value is not None else None
+            elif 'date' in column_type.lower() or 'time' in column_type.lower():
+                if isinstance(value, (datetime.datetime, pd.Timestamp)):
+                    return value.strftime('%Y-%m-%d %H:%M:%S')
+                return str(value)
+            elif 'char' in column_type.lower() or 'text' in column_type.lower():
+                return str(value)
+            else:
+                return value
+        except (ValueError, TypeError) as e:
+            raise ValueError(f"Failed to convert value {value} to type {column_type}: {str(e)}")
+
+    def _execute_with_retry(self, func, *args, **kwargs):
+        """
+        带重试机制的SQL执行装饰器
+
+        :param func: 要执行的函数
+        :param args: 位置参数
+        :param kwargs: 关键字参数
+        :return: 函数执行结果
+        """
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(self.max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except pymysql.OperationalError as e:
+                    last_exception = e
+                    if attempt < self.max_retries - 1:
+                        time.sleep(self.retry_interval)
+                        # 尝试重新连接
+                        self.pool = self._create_connection_pool()
+                        continue
+            raise last_exception if last_exception else Exception("Unknown error occurred")
+
+        return wrapper(*args, **kwargs)
+
+    def _get_connection(self):
+        """从连接池获取连接"""
+        return self.pool.connection()
+
+    def _check_database_exists(self, db_name: str) -> bool:
+        """检查数据库是否存在"""
+        db_name = self._validate_identifier(db_name)
+        sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (db_name,))
+                return bool(cursor.fetchone())
+
+    def _create_database(self, db_name: str):
+        """创建数据库"""
+        db_name = self._validate_identifier(db_name)
+        sql = f"CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql)
+            conn.commit()
+
+    def _check_table_exists(self, db_name: str, table_name: str) -> bool:
+        """检查表是否存在"""
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = """
+        SELECT TABLE_NAME 
+        FROM INFORMATION_SCHEMA.TABLES 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (db_name, table_name))
+                return bool(cursor.fetchone())
+
+    def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
+        """获取表的列名和数据类型"""
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = """
+        SELECT COLUMN_NAME, DATA_TYPE 
+        FROM INFORMATION_SCHEMA.COLUMNS 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (db_name, table_name))
+                return {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
+
+    def _create_table(
+            self,
+            db_name: str,
+            table_name: str,
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]] = None,
+            date_column: Optional[str] = None
+    ):
+        """
+        创建数据表
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param date_column: 日期列名，如果存在将设置为索引
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not columns:
+            raise ValueError("No columns specified for table creation")
+
+        # 构建列定义SQL
+        column_defs = []
+        for col_name, col_type in columns.items():
+            safe_col_name = self._validate_identifier(col_name)
+            col_def = f"`{safe_col_name}` {col_type}"
+            column_defs.append(col_def)
+
+        # 添加主键定义
+        primary_key_sql = ""
+        if primary_keys:
+            safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
+            primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+
+        # 构建完整SQL
+        sql = f"""
+        CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
+            {','.join(column_defs)}
+            {primary_key_sql}
+        ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql)
+
+            # 如果存在日期列，添加索引
+            if date_column and date_column in columns:
+                safe_date_col = self._validate_identifier(date_column)
+                index_sql = f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
+                with conn.cursor() as cursor:
+                    cursor.execute(index_sql)
+
+            conn.commit()
+
+    def _prepare_data(
+            self,
+            data: Union[Dict, List[Dict], pd.DataFrame],
+            columns: Dict[str, str],
+            allow_null: bool = False
+    ) -> List[Dict]:
+        """
+        准备要上传的数据，验证并转换数据类型
+
+        :param data: 输入数据
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param allow_null: 是否允许空值
+        :return: 准备好的数据列表
+        """
+        # 统一数据格式为字典列表
+        if isinstance(data, pd.DataFrame):
+            data = data.to_dict('records')
+        elif isinstance(data, dict):
+            data = [data]
+
+        if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
+            raise ValueError("Data must be a dict, list of dicts, or DataFrame")
+
+        prepared_data = []
+        for row in data:
+            prepared_row = {}
+            for col_name, col_type in columns.items():
+                if col_name not in row:
+                    if not allow_null:
+                        raise ValueError(f"Missing required column '{col_name}' in data")
+                    prepared_row[col_name] = None
+                else:
+                    try:
+                        prepared_row[col_name] = self._validate_value(row[col_name], col_type)
+                    except ValueError as e:
+                        raise ValueError(f"Error in column '{col_name}': {str(e)}")
+            prepared_data.append(prepared_row)
+
+        return prepared_data
+
+    def _get_partition_table_name(self, table_name: str, date_value: str, partition_by: str) -> str:
+        """
+        获取分表名称
+
+        :param table_name: 基础表名
+        :param date_value: 日期值
+        :param partition_by: 分表方式 ('year' 或 'month')
+        :return: 分表名称
+        """
+        try:
+            date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
+        except ValueError:
+            try:
+                date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
+            except ValueError:
+                raise ValueError(f"Invalid date format: {date_value}")
+
+        if partition_by == 'year':
+            return f"{table_name}_{date_obj.year}"
+        elif partition_by == 'month':
+            return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
+        else:
+            raise ValueError("partition_by must be 'year' or 'month'")
+
+    def _insert_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            batch_size: int = 1000
+    ):
+        """
+        插入数据到表中
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要插入的数据
+        :param columns: 列名和数据类型字典
+        :param check_duplicate: 是否检查重复
+        :param duplicate_columns: 用于检查重复的列列表
+        :param batch_size: 批量插入的大小
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not data:
+            return
+
+        # 获取所有列名
+        all_columns = list(columns.keys())
+        safe_columns = [self._validate_identifier(col) for col in all_columns]
+        placeholders = ','.join(['%s'] * len(safe_columns))
+
+        # 构建INSERT SQL
+        if check_duplicate:
+            if duplicate_columns:
+                # 只检查指定列的重复
+                dup_columns = [self._validate_identifier(col) for col in duplicate_columns]
+            else:
+                # 检查所有列的重复
+                dup_columns = safe_columns
+
+            # 构建ON DUPLICATE KEY UPDATE子句
+            update_clause = ','.join([f"`{col}`=VALUES(`{col}`)" for col in safe_columns])
+
+            sql = f"""
+            INSERT INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            ON DUPLICATE KEY UPDATE {update_clause}
+            """
+        else:
+            sql = f"""
+            INSERT INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            """
+
+        # 分批插入数据
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                for i in range(0, len(data), batch_size):
+                    batch = data[i:i + batch_size]
+                    # 准备批量数据
+                    values = []
+                    for row in batch:
+                        row_values = []
+                        for col in all_columns:
+                            row_values.append(row.get(col))
+                        values.append(row_values)
+
+                    # 执行批量插入
+                    try:
+                        cursor.executemany(sql, values)
+                        conn.commit()
+                    except Exception as e:
+                        conn.rollback()
+                        raise e
+
+    def upload_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: Union[Dict, List[Dict], pd.DataFrame],
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]] = None,
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            allow_null: bool = False,
+            partition_by: Optional[str] = None,
+            partition_date_column: str = '日期',
+            auto_create: bool = True
+    ):
+        """
+        上传数据到数据库
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要上传的数据
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param check_duplicate: 是否检查重复，默认为False
+        :param duplicate_columns: 用于检查重复的列列表，如果不指定则使用所有列
+        :param allow_null: 是否允许空值，默认为False
+        :param partition_by: 分表方式 ('year' 或 'month')，默认为None不分表
+        :param partition_date_column: 用于分表的日期列名，默认为'日期'
+        :param auto_create: 是否自动创建不存在的数据库或表，默认为True
+        """
+        # 验证参数
+        if not columns:
+            raise ValueError("Columns specification is required")
+
+        if partition_by and partition_by not in ['year', 'month']:
+            raise ValueError("partition_by must be 'year', 'month' or None")
+
+        # 准备数据
+        prepared_data = self._prepare_data(data, columns, allow_null)
+
+        # 检查数据库是否存在
+        if not self._check_database_exists(db_name):
+            if auto_create:
+                self._create_database(db_name)
+            else:
+                raise ValueError(f"Database '{db_name}' does not exist")
+
+        # 处理分表逻辑
+        if partition_by:
+            # 分组数据按分表
+            partitioned_data = {}
+            for row in prepared_data:
+                if partition_date_column not in row:
+                    raise ValueError(f"Partition date column '{partition_date_column}' not found in data")
+                part_table = self._get_partition_table_name(table_name, str(row[partition_date_column]), partition_by)
+                if part_table not in partitioned_data:
+                    partitioned_data[part_table] = []
+                partitioned_data[part_table].append(row)
+
+            # 对每个分表执行上传
+            for part_table, part_data in partitioned_data.items():
+                self._upload_to_table(
+                    db_name, part_table, part_data, columns,
+                    primary_keys, check_duplicate, duplicate_columns,
+                    allow_null, auto_create, partition_date_column
+                )
+        else:
+            # 不分表，直接上传
+            self._upload_to_table(
+                db_name, table_name, prepared_data, columns,
+                primary_keys, check_duplicate, duplicate_columns,
+                allow_null, auto_create, partition_date_column
+            )
+
+    def _upload_to_table(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]],
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            allow_null: bool,
+            auto_create: bool,
+            date_column: Optional[str]
+    ):
+        """实际执行表上传的内部方法"""
+        # 检查表是否存在
+        if not self._check_table_exists(db_name, table_name):
+            if auto_create:
+                self._create_table(db_name, table_name, columns, primary_keys, date_column)
+            else:
+                raise ValueError(f"Table '{db_name}.{table_name}' does not exist")
+
+        # 获取表结构并验证
+        table_columns = self._get_table_columns(db_name, table_name)
+        if not table_columns:
+            raise ValueError(f"Failed to get columns for table '{db_name}.{table_name}'")
+
+        # 验证数据列与表列匹配
+        for col in columns:
+            if col not in table_columns:
+                raise ValueError(f"Column '{col}' not found in table '{db_name}.{table_name}'")
+
+        # 插入数据
+        self._insert_data(
+            db_name, table_name, data, columns,
+            check_duplicate, duplicate_columns
+        )
+
+    def close(self):
+        """关闭连接池"""
+        self.pool.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
 class MysqlUpload:
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
         self.username = username

mdbq/spider/aikucun.py

@@ -257,13 +257,13 @@ class AikuCun:
             )
             print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
             # print(res.json())
-            if not res.json()['success']:
+            if not res.json().get('success', None):
                 print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
                 continue
-            if not res.json()['data']['rows']:
+            if not res.json().get('data', {}).get('rows', None):
                 print("返回的数据字典异常, ['data']['rows'] 不能为空")
                 num += 1
                 self.error_count += 1
@@ -479,7 +479,10 @@ def main(start_date, end_date=None, item_type=['spu']):
 
 if __name__ == '__main__':
     main(
-        start_date='2025-03-25',
-        # end_date='2025-03-26',  # 不传则默认到今天
-        item_type=['spu', 'sku']
+        start_date='2025-05-13',
+        # end_date='2025-04-28',  # 不传则默认到今天
+        item_type=[
+            # 'spu',
+            'sku'
+        ]
     )

{mdbq-3.8.18.dist-info → mdbq-3.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.18
+Version: 3.9.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.18.dist-info → mdbq-3.9.0.dist-info}/RECORD

@@ -1,13 +1,14 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=kflFjk3ujk2iECHCD3Bw3eVeZ9O3eAB76MG428NMJM0,18
+mdbq/__version__.py,sha256=7hLUrBXQAGj08UbPoot9b_BwLXRkc-RH_nJSvG9AqTc,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
+mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=umcLpw5cYGNNJnEjBLh_bgBXeh5LntPKFm8VslQ01ow,55030
+mdbq/mysql/mysql.py,sha256=L_UR7TqcZoHZj6dWVZe-ai6X2yc_oULPyUzKy7DHbOw,74493
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -20,8 +21,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=QfyUtXMuPZ5mJVNDUlFa_ltFXiCCTccBz6MT3YT-7HI,19742
-mdbq-3.8.18.dist-info/METADATA,sha256=tQFP7EyTad4nohSxELsIYAuvw8R4eQaVCxu8NDh9OD4,364
-mdbq-3.8.18.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.18.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.18.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
+mdbq-3.9.0.dist-info/METADATA,sha256=pd--meyNjH8KaX-ZgnSHqWN9GbEyx59Atb1Wgs6BqVc,363
+mdbq-3.9.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.0.dist-info/RECORD,,