mdbq 3.8.19__py3-none-any.whl → 3.9.0__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.8.19'
+VERSION = '3.9.0'

mdbq/mysql/mysql.py

@@ -12,6 +12,9 @@ import os
 import logging
 from mdbq.other import otk
 
+from dbutils.pooled_db import PooledDB
+from typing import Union, List, Dict, Optional, Any, Tuple
+
 warnings.filterwarnings('ignore')
 """
 建表流程:
@@ -43,6 +46,520 @@ def count_decimal_places(num_str):
     return 0, 0
 
 
+class MySQLUploader:
+    def __init__(
+            self,
+            username: str,
+            password: str,
+            host: str = 'localhost',
+            port: int = 3306,
+            charset: str = 'utf8mb4',
+            collation: str = 'utf8mb4_0900_ai_ci',
+            enable_logging: bool = False,
+            log_level: str = 'ERROR',
+            max_retries: int = 10,
+            retry_interval: int = 10,
+            pool_size: int = 5
+    ):
+        """
+        初始化MySQL上传工具
+
+        :param username: 数据库用户名
+        :param password: 数据库密码
+        :param host: 数据库主机地址，默认为localhost
+        :param port: 数据库端口，默认为3306
+        :param charset: 字符集，默认为utf8mb4
+        :param collation: 排序规则，默认为utf8mb4_0900_ai_ci
+        :param enable_logging: 是否启用日志，默认为False
+        :param log_level: 日志级别，默认为ERROR
+        :param max_retries: 最大重试次数，默认为10
+        :param retry_interval: 重试间隔(秒)，默认为10
+        :param pool_size: 连接池大小，默认为5
+        """
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+        self.charset = charset
+        self.collation = collation
+        self.max_retries = max_retries
+        self.retry_interval = retry_interval
+        self.pool_size = pool_size
+
+        # 初始化日志
+        if enable_logging:
+            self._init_logging(log_level)
+
+        # 创建连接池
+        self.pool = self._create_connection_pool()
+
+    def _init_logging(self, log_level: str):
+        """初始化日志配置"""
+        logging.basicConfig(
+            level=getattr(logging, log_level.upper(), logging.ERROR),
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        self.logger = logging.getLogger('MySQLUploader')
+
+    def _create_connection_pool(self):
+        """创建数据库连接池"""
+        return PooledDB(
+            creator=pymysql,
+            host=self.host,
+            port=self.port,
+            user=self.username,
+            password=self.password,
+            charset=self.charset,
+            maxconnections=self.pool_size,
+            cursorclass=pymysql.cursors.DictCursor
+        )
+
+    def _validate_identifier(self, identifier: str) -> str:
+        """
+        验证并清理数据库标识符(数据库名、表名、列名)
+        防止SQL注入和非法字符
+
+        :param identifier: 要验证的标识符
+        :return: 清理后的安全标识符
+        """
+        if not identifier or not isinstance(identifier, str):
+            raise ValueError(f"Invalid identifier: {identifier}")
+
+        # 移除可能有害的字符
+        cleaned = re.sub(r'[^a-zA-Z0-9_$]', '', identifier)
+        if not cleaned:
+            raise ValueError(f"Invalid identifier after cleaning: {identifier}")
+
+        return cleaned
+
+    def _validate_value(self, value: Any, column_type: str) -> Any:
+        """
+        验证并清理数据值，根据列类型进行适当转换
+
+        :param value: 要验证的值
+        :param column_type: 列的数据类型
+        :return: 清理后的值
+        """
+        if value is None:
+            return None
+
+        try:
+            if 'int' in column_type.lower():
+                return int(value) if value is not None else None
+            elif 'float' in column_type.lower() or 'double' in column_type.lower() or 'decimal' in column_type.lower():
+                return float(value) if value is not None else None
+            elif 'date' in column_type.lower() or 'time' in column_type.lower():
+                if isinstance(value, (datetime.datetime, pd.Timestamp)):
+                    return value.strftime('%Y-%m-%d %H:%M:%S')
+                return str(value)
+            elif 'char' in column_type.lower() or 'text' in column_type.lower():
+                return str(value)
+            else:
+                return value
+        except (ValueError, TypeError) as e:
+            raise ValueError(f"Failed to convert value {value} to type {column_type}: {str(e)}")
+
+    def _execute_with_retry(self, func, *args, **kwargs):
+        """
+        带重试机制的SQL执行装饰器
+
+        :param func: 要执行的函数
+        :param args: 位置参数
+        :param kwargs: 关键字参数
+        :return: 函数执行结果
+        """
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(self.max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except pymysql.OperationalError as e:
+                    last_exception = e
+                    if attempt < self.max_retries - 1:
+                        time.sleep(self.retry_interval)
+                        # 尝试重新连接
+                        self.pool = self._create_connection_pool()
+                        continue
+            raise last_exception if last_exception else Exception("Unknown error occurred")
+
+        return wrapper(*args, **kwargs)
+
+    def _get_connection(self):
+        """从连接池获取连接"""
+        return self.pool.connection()
+
+    def _check_database_exists(self, db_name: str) -> bool:
+        """检查数据库是否存在"""
+        db_name = self._validate_identifier(db_name)
+        sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (db_name,))
+                return bool(cursor.fetchone())
+
+    def _create_database(self, db_name: str):
+        """创建数据库"""
+        db_name = self._validate_identifier(db_name)
+        sql = f"CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql)
+            conn.commit()
+
+    def _check_table_exists(self, db_name: str, table_name: str) -> bool:
+        """检查表是否存在"""
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = """
+        SELECT TABLE_NAME 
+        FROM INFORMATION_SCHEMA.TABLES 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (db_name, table_name))
+                return bool(cursor.fetchone())
+
+    def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
+        """获取表的列名和数据类型"""
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = """
+        SELECT COLUMN_NAME, DATA_TYPE 
+        FROM INFORMATION_SCHEMA.COLUMNS 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (db_name, table_name))
+                return {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
+
+    def _create_table(
+            self,
+            db_name: str,
+            table_name: str,
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]] = None,
+            date_column: Optional[str] = None
+    ):
+        """
+        创建数据表
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param date_column: 日期列名，如果存在将设置为索引
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not columns:
+            raise ValueError("No columns specified for table creation")
+
+        # 构建列定义SQL
+        column_defs = []
+        for col_name, col_type in columns.items():
+            safe_col_name = self._validate_identifier(col_name)
+            col_def = f"`{safe_col_name}` {col_type}"
+            column_defs.append(col_def)
+
+        # 添加主键定义
+        primary_key_sql = ""
+        if primary_keys:
+            safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
+            primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+
+        # 构建完整SQL
+        sql = f"""
+        CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
+            {','.join(column_defs)}
+            {primary_key_sql}
+        ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql)
+
+            # 如果存在日期列，添加索引
+            if date_column and date_column in columns:
+                safe_date_col = self._validate_identifier(date_column)
+                index_sql = f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
+                with conn.cursor() as cursor:
+                    cursor.execute(index_sql)
+
+            conn.commit()
+
+    def _prepare_data(
+            self,
+            data: Union[Dict, List[Dict], pd.DataFrame],
+            columns: Dict[str, str],
+            allow_null: bool = False
+    ) -> List[Dict]:
+        """
+        准备要上传的数据，验证并转换数据类型
+
+        :param data: 输入数据
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param allow_null: 是否允许空值
+        :return: 准备好的数据列表
+        """
+        # 统一数据格式为字典列表
+        if isinstance(data, pd.DataFrame):
+            data = data.to_dict('records')
+        elif isinstance(data, dict):
+            data = [data]
+
+        if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
+            raise ValueError("Data must be a dict, list of dicts, or DataFrame")
+
+        prepared_data = []
+        for row in data:
+            prepared_row = {}
+            for col_name, col_type in columns.items():
+                if col_name not in row:
+                    if not allow_null:
+                        raise ValueError(f"Missing required column '{col_name}' in data")
+                    prepared_row[col_name] = None
+                else:
+                    try:
+                        prepared_row[col_name] = self._validate_value(row[col_name], col_type)
+                    except ValueError as e:
+                        raise ValueError(f"Error in column '{col_name}': {str(e)}")
+            prepared_data.append(prepared_row)
+
+        return prepared_data
+
+    def _get_partition_table_name(self, table_name: str, date_value: str, partition_by: str) -> str:
+        """
+        获取分表名称
+
+        :param table_name: 基础表名
+        :param date_value: 日期值
+        :param partition_by: 分表方式 ('year' 或 'month')
+        :return: 分表名称
+        """
+        try:
+            date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
+        except ValueError:
+            try:
+                date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
+            except ValueError:
+                raise ValueError(f"Invalid date format: {date_value}")
+
+        if partition_by == 'year':
+            return f"{table_name}_{date_obj.year}"
+        elif partition_by == 'month':
+            return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
+        else:
+            raise ValueError("partition_by must be 'year' or 'month'")
+
+    def _insert_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            batch_size: int = 1000
+    ):
+        """
+        插入数据到表中
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要插入的数据
+        :param columns: 列名和数据类型字典
+        :param check_duplicate: 是否检查重复
+        :param duplicate_columns: 用于检查重复的列列表
+        :param batch_size: 批量插入的大小
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+
+        if not data:
+            return
+
+        # 获取所有列名
+        all_columns = list(columns.keys())
+        safe_columns = [self._validate_identifier(col) for col in all_columns]
+        placeholders = ','.join(['%s'] * len(safe_columns))
+
+        # 构建INSERT SQL
+        if check_duplicate:
+            if duplicate_columns:
+                # 只检查指定列的重复
+                dup_columns = [self._validate_identifier(col) for col in duplicate_columns]
+            else:
+                # 检查所有列的重复
+                dup_columns = safe_columns
+
+            # 构建ON DUPLICATE KEY UPDATE子句
+            update_clause = ','.join([f"`{col}`=VALUES(`{col}`)" for col in safe_columns])
+
+            sql = f"""
+            INSERT INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            ON DUPLICATE KEY UPDATE {update_clause}
+            """
+        else:
+            sql = f"""
+            INSERT INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+            """
+
+        # 分批插入数据
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                for i in range(0, len(data), batch_size):
+                    batch = data[i:i + batch_size]
+                    # 准备批量数据
+                    values = []
+                    for row in batch:
+                        row_values = []
+                        for col in all_columns:
+                            row_values.append(row.get(col))
+                        values.append(row_values)
+
+                    # 执行批量插入
+                    try:
+                        cursor.executemany(sql, values)
+                        conn.commit()
+                    except Exception as e:
+                        conn.rollback()
+                        raise e
+
+    def upload_data(
+            self,
+            db_name: str,
+            table_name: str,
+            data: Union[Dict, List[Dict], pd.DataFrame],
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]] = None,
+            check_duplicate: bool = False,
+            duplicate_columns: Optional[List[str]] = None,
+            allow_null: bool = False,
+            partition_by: Optional[str] = None,
+            partition_date_column: str = '日期',
+            auto_create: bool = True
+    ):
+        """
+        上传数据到数据库
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要上传的数据
+        :param columns: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表
+        :param check_duplicate: 是否检查重复，默认为False
+        :param duplicate_columns: 用于检查重复的列列表，如果不指定则使用所有列
+        :param allow_null: 是否允许空值，默认为False
+        :param partition_by: 分表方式 ('year' 或 'month')，默认为None不分表
+        :param partition_date_column: 用于分表的日期列名，默认为'日期'
+        :param auto_create: 是否自动创建不存在的数据库或表，默认为True
+        """
+        # 验证参数
+        if not columns:
+            raise ValueError("Columns specification is required")
+
+        if partition_by and partition_by not in ['year', 'month']:
+            raise ValueError("partition_by must be 'year', 'month' or None")
+
+        # 准备数据
+        prepared_data = self._prepare_data(data, columns, allow_null)
+
+        # 检查数据库是否存在
+        if not self._check_database_exists(db_name):
+            if auto_create:
+                self._create_database(db_name)
+            else:
+                raise ValueError(f"Database '{db_name}' does not exist")
+
+        # 处理分表逻辑
+        if partition_by:
+            # 分组数据按分表
+            partitioned_data = {}
+            for row in prepared_data:
+                if partition_date_column not in row:
+                    raise ValueError(f"Partition date column '{partition_date_column}' not found in data")
+                part_table = self._get_partition_table_name(table_name, str(row[partition_date_column]), partition_by)
+                if part_table not in partitioned_data:
+                    partitioned_data[part_table] = []
+                partitioned_data[part_table].append(row)
+
+            # 对每个分表执行上传
+            for part_table, part_data in partitioned_data.items():
+                self._upload_to_table(
+                    db_name, part_table, part_data, columns,
+                    primary_keys, check_duplicate, duplicate_columns,
+                    allow_null, auto_create, partition_date_column
+                )
+        else:
+            # 不分表，直接上传
+            self._upload_to_table(
+                db_name, table_name, prepared_data, columns,
+                primary_keys, check_duplicate, duplicate_columns,
+                allow_null, auto_create, partition_date_column
+            )
+
+    def _upload_to_table(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            columns: Dict[str, str],
+            primary_keys: Optional[List[str]],
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            allow_null: bool,
+            auto_create: bool,
+            date_column: Optional[str]
+    ):
+        """实际执行表上传的内部方法"""
+        # 检查表是否存在
+        if not self._check_table_exists(db_name, table_name):
+            if auto_create:
+                self._create_table(db_name, table_name, columns, primary_keys, date_column)
+            else:
+                raise ValueError(f"Table '{db_name}.{table_name}' does not exist")
+
+        # 获取表结构并验证
+        table_columns = self._get_table_columns(db_name, table_name)
+        if not table_columns:
+            raise ValueError(f"Failed to get columns for table '{db_name}.{table_name}'")
+
+        # 验证数据列与表列匹配
+        for col in columns:
+            if col not in table_columns:
+                raise ValueError(f"Column '{col}' not found in table '{db_name}.{table_name}'")
+
+        # 插入数据
+        self._insert_data(
+            db_name, table_name, data, columns,
+            check_duplicate, duplicate_columns
+        )
+
+    def close(self):
+        """关闭连接池"""
+        self.pool.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
 class MysqlUpload:
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
         self.username = username

mdbq/spider/aikucun.py

@@ -257,13 +257,13 @@ class AikuCun:
             )
             print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
             # print(res.json())
-            if not res.json()['success']:
+            if not res.json().get('success', None):
                 print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
                 continue
-            if not res.json()['data']['rows']:
+            if not res.json().get('data', {}).get('rows', None):
                 print("返回的数据字典异常, ['data']['rows'] 不能为空")
                 num += 1
                 self.error_count += 1
@@ -479,7 +479,10 @@ def main(start_date, end_date=None, item_type=['spu']):
 
 if __name__ == '__main__':
     main(
-        start_date='2025-03-25',
-        # end_date='2025-03-26',  # 不传则默认到今天
-        item_type=['spu', 'sku']
+        start_date='2025-05-13',
+        # end_date='2025-04-28',  # 不传则默认到今天
+        item_type=[
+            # 'spu',
+            'sku'
+        ]
     )

{mdbq-3.8.19.dist-info → mdbq-3.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.19
+Version: 3.9.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.19.dist-info → mdbq-3.9.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=adbn7k6HeaPn3waOOAb2wMca7SSJxTRZ8MEbtLkPAeA,18
+mdbq/__version__.py,sha256=7hLUrBXQAGj08UbPoot9b_BwLXRkc-RH_nJSvG9AqTc,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -8,7 +8,7 @@ mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=umcLpw5cYGNNJnEjBLh_bgBXeh5LntPKFm8VslQ01ow,55030
+mdbq/mysql/mysql.py,sha256=L_UR7TqcZoHZj6dWVZe-ai6X2yc_oULPyUzKy7DHbOw,74493
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -21,8 +21,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=QfyUtXMuPZ5mJVNDUlFa_ltFXiCCTccBz6MT3YT-7HI,19742
-mdbq-3.8.19.dist-info/METADATA,sha256=yheVi8Il8o8Py_HUlKc8s-gCako_d8RktLqdNa-TPSA,364
-mdbq-3.8.19.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.19.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.19.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
+mdbq-3.9.0.dist-info/METADATA,sha256=pd--meyNjH8KaX-ZgnSHqWN9GbEyx59Atb1Wgs6BqVc,363
+mdbq-3.9.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.0.dist-info/RECORD,,