mdbq 4.0.9__py3-none-any.whl → 4.0.11__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.9'
+VERSION = '4.0.11'

mdbq/aggregation/query_data.py

@@ -1557,7 +1557,6 @@ class MysqlDatasQuery:
             'unique_keys': [['日期', '店铺id', '商品id']],  # 唯一约束列表
         }
 
-
     @upload_data_decorator()
     def spph(self, db_name='聚合数据', table_name='天猫_商品排行'):
         """  """
@@ -3677,6 +3676,7 @@ def query3(months=1, download_manager=None):
 
 
 def main(months=3):
+    logger.info('数据聚合任务开始')
     # 1. 更新日期表  更新货品年份基准表， 属性设置 3 - 货品年份基准
     date_table()
     # 2. 数据聚合
@@ -3685,11 +3685,12 @@ def main(months=3):
         password=password, 
         host=host, 
         port=port,
-        maxconnections=30,
+        maxconnections=20,
         )
     query1(download_manager=download_manager, months=months)
     query2(download_manager=download_manager, months=months)
     query3(download_manager=download_manager, months=months)
+    logger.info('数据聚合完成')
 
 
 if __name__ == '__main__':

mdbq/mysql/deduplicator.py

@@ -6,6 +6,7 @@ import warnings
 import pymysql
 import os
 from mdbq.log import mylogger
+from mdbq.config import config
 from typing import List, Dict, Optional, Any, Tuple
 from dbutils.pooled_db import PooledDB
 import threading
@@ -1348,7 +1349,7 @@ class MySQLDeduplicator:
 
 
 def main():
-    from mdbq.config import config
+    logger.info('去重任务开始')
     dir_path = os.path.expanduser("~")
     my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
     username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
@@ -1401,6 +1402,8 @@ def main():
 
     # 关闭连接
     deduplicator.close()
+    logger.info('去重任务结束')
+
 
 if __name__ == '__main__':
     main()

mdbq/mysql/s_query.py

@@ -35,7 +35,7 @@ class QueryDatas:
     """
 
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4',
-                 maxconnections: int = 20, mincached: int = 2, maxcached: int = 5,
+                 pool_size: int = 20, mincached: int = 2, maxcached: int = 5,
                  connect_timeout: int = 10, read_timeout: int = 30, write_timeout: int = 30,
                  max_retries: int = 3, retry_waiting_time: int = 5, collation: str = 'utf8mb4_0900_ai_ci') -> None:
         """
@@ -47,9 +47,9 @@ class QueryDatas:
             host: 数据库主机
             port: 数据库端口
             charset: 字符集，默认utf8mb4
-            maxconnections: 最大连接数，默认20
-            mincached: 最小缓存连接数，默认2
-            maxcached: 最大缓存连接数，默认5
+            pool_size: 最大活动连接数，默认20
+            mincached: 最小缓存连接数，空闲连接数量，默认2
+            maxcached: 最大缓存连接数，最大空闲连接数，默认5
             connect_timeout: 连接超时时间，默认10秒
             read_timeout: 读取超时时间，默认30秒
             write_timeout: 写入超时时间，默认30秒
@@ -87,14 +87,14 @@ class QueryDatas:
             'write_timeout': write_timeout,
             'autocommit': True
         }
-        self.pool = self._create_connection_pool(maxconnections, mincached, maxcached)
+        self.pool = self._create_connection_pool(pool_size, mincached, maxcached)
 
-    def _create_connection_pool(self, maxconnections: int, mincached: int, maxcached: int) -> PooledDB:
+    def _create_connection_pool(self, pool_size: int, mincached: int, maxcached: int) -> PooledDB:
         """
         创建数据库连接池
         
         Args:
-            maxconnections: 最大连接数
+            pool_size: 最大连接数
             mincached: 最小缓存连接数
             maxcached: 最大缓存连接数
             
@@ -122,7 +122,7 @@ class QueryDatas:
         }
         pool_params = {
             'creator': pymysql,
-            'maxconnections': maxconnections,
+            'maxconnections': pool_size,
             'mincached': mincached,
             'maxcached': maxcached,
             'blocking': True,
@@ -133,7 +133,7 @@ class QueryDatas:
         try:
             pool = PooledDB(**pool_params, **connection_params)
             logger.debug('连接池创建成功', {
-                '连接池大小': maxconnections,
+                '连接池大小': pool_size,
                 '最小缓存': mincached,
                 '最大缓存': maxcached,
                 '主机': self.host,
@@ -253,20 +253,8 @@ class QueryDatas:
 
     # @_execute_with_retry
     def _get_connection(self, db_name: Optional[str] = None) -> pymysql.connections.Connection:
-        """
-        从连接池获取数据库连接
-        
-        Args:
-            db_name: 可选的数据库名，如果提供则会在连接后选择该数据库
-            
-        Returns:
-            数据库连接对象
-            
-        Raises:
-            ConnectionError: 当获取连接失败时抛出
-        """
+        """从连接池获取数据库连接"""
         try:
-            # 只在连续失败次数达到阈值时检查健康状态
             if self._pool_stats['consecutive_failures'] >= self._pool_stats['max_consecutive_failures']:
                 if not self._check_pool_health():
                     logger.warning('连接池不健康，尝试重新创建')
@@ -282,66 +270,184 @@ class QueryDatas:
             error_code = e.args[0] if e.args else None
             if error_code in (2003, 2006, 2013):
                 logger.error('数据库连接错误', {
+                    '库': db_name,
                     '错误代码': error_code,
                     '错误信息': str(e),
-                    '数据库': db_name
                 })
                 self.pool = self._create_connection_pool(10, 2, 5)
                 self._pool_stats['consecutive_failures'] = 0
                 raise ConnectionError(f'数据库连接错误: {str(e)}')
-            else:
-                raise
+            raise
         except Exception as e:
             logger.error('从连接池获取数据库连接失败', {
+                '库': db_name,
                 '错误': str(e),
-                '数据库': db_name
             })
             raise ConnectionError(f'连接数据库失败: {str(e)}')
 
     # @_execute_with_retry
-    def _execute_query(self, sql: str, params: tuple = None, db_name: str = None) -> Optional[List[Dict[str, Any]]]:
+    def _execute_query(self, sql: str, params: tuple = None, db_name: str = None, 
+                      fetch_all: bool = True, error_handling: bool = True) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
+        """执行SQL查询的通用方法"""
+        try:
+            if sql.upper().startswith('SHOW DATABASES'):
+                with closing(self._get_connection()) as connection:
+                    with closing(connection.cursor()) as cursor:
+                        cursor.execute(sql, params)
+                        return cursor.fetchall() if fetch_all else cursor.fetchone()
+            else:
+                with closing(self._get_connection(db_name)) as connection:
+                    with closing(connection.cursor()) as cursor:
+                        cursor.execute(sql, params)
+                        return cursor.fetchall() if fetch_all else cursor.fetchone()
+        except pymysql.OperationalError as e:
+            error_code = e.args[0] if e.args else None
+            if error_handling:
+                if error_code in (1045, 1049):  # 访问被拒绝或数据库不存在
+                    logger.error('数据库访问错误', {
+                        'SQL': sql,
+                        '参数': params,
+                        '库': db_name,
+                        '错误代码': error_code,
+                        '错误信息': str(e)
+                    })
+                else:
+                    logger.error('数据库操作错误', {
+                        '库': db_name,
+                        'SQL': sql,
+                        '参数': params,
+                        '错误代码': error_code,
+                        '错误信息': str(e)
+                    })
+                return None
+            raise
+        except Exception as e:
+            if error_handling:
+                logger.error('执行SQL查询失败', {
+                    '库': db_name,
+                    'SQL': sql,
+                    '参数': params,
+                    '错误类型': type(e).__name__,
+                    '错误信息': str(e)
+                })
+                return None
+            raise
+
+    def _get_table_info(self, db_name: str, table_name: str, info_type: Literal['columns', 'dtypes', 'exists'] = 'exists') -> Union[bool, List[Dict[str, Any]], List[str]]:
         """
-        执行SQL查询的通用方法。
+        获取表信息的通用方法。
         
         Args:
-            sql: SQL查询语句
-            params: 查询参数
             db_name: 数据库名
-            
+            table_name: 表名
+            info_type: 信息类型
+                - 'exists': 检查表是否存在（默认）
+                - 'columns': 获取列名列表
+                - 'dtypes': 获取列名和类型
+                
         Returns:
-            查询结果列表，如果查询失败返回None
+            根据info_type返回不同类型的信息：
+            - 'exists': 返回bool，表示表是否存在
+            - 'columns': 返回列名列表
+            - 'dtypes': 返回列名和类型的列表
         """
         try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    cursor.execute(sql, params)
-                    return cursor.fetchall()
+            if info_type == 'exists':
+                result = self._execute_query("SHOW DATABASES LIKE %s", (db_name,))
+                if not result:
+                    all_dbs = self._execute_query("SHOW DATABASES")
+                    available_dbs = [db['Database'] for db in all_dbs] if all_dbs else []
+                    logger.info('数据库不存在', {
+                        '库': db_name,
+                        '可用的数据库': available_dbs,
+                        '可能的原因': '数据库名称错误或没有访问权限'
+                    })
+                    return False
+                    
+                result = self._execute_query("SHOW TABLES LIKE %s", (table_name,), db_name=db_name)
+                if not result:
+                    all_tables = self._execute_query("SHOW TABLES", db_name=db_name)
+                    available_tables = [table[f'Tables_in_{db_name}'] for table in all_tables] if all_tables else []
+                    logger.info('表不存在', {
+                        '库': db_name,
+                        '表': table_name,
+                        '可用的表': available_tables,
+                        '可能的原因': '表名称错误或没有访问权限'
+                    })
+                    return False
+                return True
+                
+            elif info_type == 'columns':
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                result = self._execute_query(sql, (db_name, table_name))
+                return [col['COLUMN_NAME'] for col in result] if result else []
+                
+            elif info_type == 'dtypes':
+                sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                return self._execute_query(sql, (db_name, table_name)) or []
+                
         except Exception as e:
-            logger.error('执行SQL查询失败', {
-                'SQL': sql,
-                '参数': params,
-                '数据库': db_name,
+            logger.error('获取表信息失败', {
+                '库': db_name,
+                '表': table_name,
+                '信息类型': info_type,
                 '错误类型': type(e).__name__,
                 '错误信息': str(e)
             })
-            return None
+            return [] if info_type != 'exists' else False
 
-    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
-        """
-        按指定条件查询数据库表，返回满足条件的指定字段数据。
+    def check_infos(self, db_name: str, table_name: str) -> bool:
+        """检查数据库和数据表是否存在"""
+        return self._get_table_info(db_name, table_name, 'exists')
+
+    def _format_columns(self, columns: List[str]) -> str:
+        """格式化列名列表为SQL语句"""
+        return ', '.join([f'`{col}`' for col in columns])
+
+    def columns_to_list(self, db_name: str, table_name: str, columns_name: List[str], where: str = None) -> List[Dict[str, Any]]:
+        """获取数据表的指定列数据"""
+        if not self._get_table_info(db_name, table_name):
+            return []
         
-        Args:
-            db_name: 数据库名
-            table_name: 表名
-            condition: SQL条件字符串（不含WHERE）
-            columns: 查询字段字符串或以逗号分隔的字段名，默认'更新时间'
+        try:
+            existing_columns = self._get_table_info(db_name, table_name, 'columns')
+            columns_name = [col for col in columns_name if col in existing_columns]
             
-        Returns:
-            查询结果列表，如果查询失败返回None
-        """
-        if not self.check_infos(db_name, table_name):
-            return None
+            if not columns_name:
+                logger.info('未找到匹配的列名', {'库': db_name, '表': table_name, '请求列': columns_name})
+                return []
+            
+            sql = f"SELECT {self._format_columns(columns_name)} FROM `{db_name}`.`{table_name}`"
+            if where:
+                sql += f" WHERE {where}"
             
+            logger.debug('执行列查询', {'库': db_name, '表': table_name, 'SQL': sql})
+            return self._execute_query(sql, db_name=db_name) or []
+            
+        except Exception as e:
+            logger.error('列查询失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
+            return []
+
+    def dtypes_to_list(self, db_name: str, table_name: str, columns_name: List[str] = None) -> List[Dict[str, Any]]:
+        """获取数据表的列名和类型"""
+        if not self._get_table_info(db_name, table_name):
+            return []
+        
+        try:
+            result = self._get_table_info(db_name, table_name, 'dtypes')
+            if columns_name:
+                columns_name = set(columns_name)
+                result = [row for row in result if row['COLUMN_NAME'] in columns_name]
+            return result
+        except Exception as e:
+            logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
+            return []
+
+    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
+        """按指定条件查询数据库表"""
+        if not self._get_table_info(db_name, table_name):
+            return None
+        
         sql = f"SELECT {columns} FROM `{table_name}` WHERE {condition}"
         logger.debug('执行SQL查询', {'库': db_name, '表': table_name, 'SQL': sql})
         return self._execute_query(sql, db_name=db_name)
@@ -598,98 +704,6 @@ class QueryDatas:
                 df[col] = df[col].astype(float)
         return df
 
-    # @_execute_with_retry
-    def columns_to_list(self, db_name, table_name, columns_name, where: str = None) -> list:
-        """
-        获取数据表的指定列, 支持where条件筛选, 返回列表字典。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns_name: 需要获取的列名列表
-        :param where: 可选，SQL条件字符串（不含WHERE）
-        :return: [{列1:值, 列2:值, ...}, ...]
-        """
-        if not self.check_infos(db_name, table_name):
-            return []
-            
-        try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, table_name))
-                    cols_exist = [col['COLUMN_NAME'] for col in cursor.fetchall()]
-                    columns_name = [item for item in columns_name if item in cols_exist]
-                    if not columns_name:
-                        logger.info('未找到匹配的列名', {'库': db_name, '表': table_name, '请求列': columns_name})
-                        return []
-                    columns_in = ', '.join([f'`{col}`' for col in columns_name])
-                    sql = f"SELECT {columns_in} FROM `{db_name}`.`{table_name}`"
-                    if where:
-                        sql += f" WHERE {where}"
-                    logger.debug('执行列查询', {'库': db_name, '表': table_name, 'SQL': sql})
-                    cursor.execute(sql)
-                    column_values = cursor.fetchall()
-            return column_values
-        except Exception as e:
-            logger.error('列查询失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
-            return []
-
-    # @_execute_with_retry
-    def dtypes_to_list(self, db_name, table_name, columns_name=None) -> list:
-        """
-        获取数据表的列名和类型, 支持只返回部分字段类型。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns_name: 可选，字段名列表，仅返回这些字段的类型
-        :return: [{'COLUMN_NAME': ..., 'COLUMN_TYPE': ...}, ...]
-        """
-        if not self.check_infos(db_name, table_name):
-            return []
-            
-        try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, table_name))
-                    column_name_and_type = cursor.fetchall()
-                    if columns_name:
-                        columns_name = set(columns_name)
-                        column_name_and_type = [row for row in column_name_and_type if row['COLUMN_NAME'] in columns_name]
-            return column_name_and_type
-        except Exception as e:
-            logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
-            return []
-
-    # @_execute_with_retry
-    def check_infos(self, db_name, table_name) -> bool:
-        """
-        检查数据库和数据表是否存在。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :return: 存在返回True，否则False
-        """
-        try:
-            # 检查数据库是否存在
-            result = self._execute_query("SHOW DATABASES LIKE %s", (db_name,))
-            if not result:
-                logger.info('数据库不存在', {'库': db_name})
-                return False
-                
-            # 检查表是否存在
-            result = self._execute_query("SHOW TABLES LIKE %s", (table_name,), db_name=db_name)
-            if not result:
-                logger.info('表不存在', {'库': db_name, '表': table_name})
-                return False
-            return True
-            
-        except Exception as e:
-            logger.error('检查数据库或表失败', {
-                '库': db_name,
-                '表': table_name,
-                '错误类型': type(e).__name__,
-                '错误信息': str(e)
-            })
-            return False
-
     def __enter__(self):
         """上下文管理器入口"""
         return self
@@ -703,7 +717,7 @@ class QueryDatas:
         if hasattr(self, 'pool') and self.pool is not None:
             try:
                 self.pool.close()
-                logger.info('连接池已关闭', {
+                logger.debug('连接池已关闭', {
                     '主机': self.host,
                     '端口': self.port
                 })
@@ -772,6 +786,8 @@ class QueryDatas:
             - 当return_format='list_dict'时，返回列表字典
             - 如果查询失败，返回空的DataFrame或空列表
         """
+        start_time = time.time()
+        
         if not db_name or not table_name:
             logger.error('数据库名和表名不能为空', {'库': db_name, '表': table_name})
             return [] if return_format == 'list_dict' else pd.DataFrame()
@@ -786,7 +802,7 @@ class QueryDatas:
         start_date, end_date = self._validate_date_range(start_date, end_date, db_name, table_name)
         
         # 检查数据库和表是否存在
-        if not self.check_infos(db_name, table_name):
+        if not self._get_table_info(db_name, table_name):
             return [] if return_format == 'list_dict' else pd.DataFrame()
         try:
             with closing(self._get_connection(db_name)) as connection:
@@ -863,7 +879,7 @@ class QueryDatas:
                     target_time = 1.0  # 期望每批1秒
 
                     while offset < total_count:
-                        start_time = time.time()
+                        _p_time = time.time()
                         # 添加分页参数
                         page_sql = f"{base_sql} LIMIT %s OFFSET %s"
                         page_params = list(params) + [page_size, offset]
@@ -881,7 +897,7 @@ class QueryDatas:
                             else:
                                 all_results = pd.concat([all_results, pd.DataFrame(page_results)], ignore_index=True)
 
-                        duration = time.time() - start_time
+                        duration = time.time() - _p_time
                         page_size = self._adjust_page_size(duration, page_size, min_size, max_size, target_time)
                         offset += len(page_results)
                         logger.debug('分页查询进度', {
@@ -896,6 +912,21 @@ class QueryDatas:
 
                     if return_format == 'df' and isinstance(all_results, pd.DataFrame) and not all_results.empty:
                         all_results = self._convert_decimal_columns(all_results)
+                    logger.info('查询完成', {
+                        '库': db_name,
+                        '表': table_name,
+                        '总记录数': total_count,
+                        '已获取记录数': len(all_results) if return_format == 'list_dict' else len(all_results.index),
+                        '查询耗时': f'{time.time() - start_time:.2f}s',
+                        '查询参数': {
+                            '开始日期': start_date,
+                            '结束日期': end_date,
+                            '日期字段': date_field,
+                            '限制行数': limit,
+                            '分页大小': page_size,
+                            '返回数据格式': return_format,
+                        }
+                    })
                     return all_results
 
         except Exception as e:
@@ -903,7 +934,15 @@ class QueryDatas:
                 '库': db_name,
                 '表': table_name,
                 '错误类型': type(e).__name__,
-                '错误信息': str(e)
+                '错误信息': str(e),
+                '查询参数': {
+                    '开始日期': start_date,
+                    '结束日期': end_date,
+                    '日期字段': date_field,
+                    '限制行数': limit,
+                    '分页大小': page_size,
+                    '返回数据格式': return_format,
+                }
             })
             return [] if return_format == 'list_dict' else pd.DataFrame()
 

mdbq/mysql/uploader.py

@@ -539,6 +539,8 @@ class MySQLUploader:
             is_nan = True
         elif str(value).lower() in ['nan', 'none']:
             is_nan = True
+        elif value == '':
+            is_nan = True
         if is_nan:
             if not allow_null:
                 if 'int' in column_type_lower:

mdbq/spider/aikucun.py

@@ -25,9 +25,7 @@ config_file = os.path.join(dir_path, 'spd.txt')
 content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
-uld = uploader.MySQLUploader(username=username, password=password, host=host, port=int(port), pool_size=10)
 # 实例化一个数据查询类，用来获取 cookies 表数据
-download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 logger = mylogger.MyLogger(
     logging_mode='file',
     log_level='info',
@@ -48,15 +46,15 @@ def keep_connect(_db_name, _config, max_try: int=10):
             connection = pymysql.connect(**_config)  # 连接数据库
             return connection
         except Exception as e:
-            logger.error(f'{_db_name}: 连接失败，正在重试: {host}:{port}  {attempts}/{max_try} {e}')
+            logger.error('连接失败', {'数据库': _db_name, '主机': host, '端口': port, '重试次数': attempts, '最大重试次数': max_try, '错误信息': e})
             attempts += 1
             time.sleep(30)
-    logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
+    logger.error('连接失败', {'数据库': _db_name, '主机': host, '端口': port, '重试次数': attempts, '最大重试次数': max_try})
     return None
 
 
 class AikuCun:
-    def __init__(self):
+    def __init__(self, uld_manager, download_manager):
         self.url = 'https://gray-merc.aikucun.com/index.html'
         self.db_name = 'cookie文件'
         self.table_name = 'main_aikucun'
@@ -66,6 +64,8 @@ class AikuCun:
         self.start_date = (self.today - datetime.timedelta(days=7)).strftime('%Y-%m-%d')
         self.end_date = (self.today - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
         self.error_count = 0
+        self.uld = uld_manager
+        self.download = download_manager
 
     def logining(self, shop_name='aikucun', headless=False):
         option = webdriver.ChromeOptions()
@@ -171,7 +171,7 @@ class AikuCun:
 
     def save_token(self):
         if not self.token:
-            print('self.token 不能为空')
+            logger.error('self.token 不能为空')
             return
         set_typ = {
             '日期': 'DATE',
@@ -182,11 +182,11 @@ class AikuCun:
             '更新时间': 'timestamp'
         }
         # 更新至数据库记录
-        uld.upload_data(
+        self.uld.upload_data(
             db_name=self.db_name,
             table_name=self.table_name,
             data=self.token,
-            set_typ={},
+            set_typ=set_typ,
             primary_keys=[],
             check_duplicate=False,
             update_on_duplicate=False,
@@ -209,7 +209,7 @@ class AikuCun:
             self.end_date = end_date
         date_list = otk.dates_between(start_date=self.start_date, end_date=self.end_date)
 
-        df = download.data_to_df(
+        df = self.download.data_to_df(
             db_name=self.db_name,
             table_name=self.table_name,
             start_date='2025-03-07',
@@ -230,7 +230,7 @@ class AikuCun:
             idx = df.groupby(['平台', '店铺名称'])['更新时间'].idxmax()
             df = df.loc[idx][['token']]
             if len(df) == 0:
-                print(f'从数据库获取的 token 不能为空')
+                logger.error(f'从数据库获取的 token 不能为空')
                 return
             self.token = df.iloc[0, 0]
 
@@ -247,7 +247,7 @@ class AikuCun:
         results = []
         for date in date_list:
             if self.error_count > 5:
-                print('已退出请求 -> self.error_count > 5')
+                logger.logger('已退出请求 -> self.error_count > 5')
                 break
             req_date = re.sub('-', '', date)
             data = {
@@ -273,16 +273,15 @@ class AikuCun:
                 # cookies=cookies,
                 data=json.dumps(data)
             )
-            print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单  {date}')
-            # print(res.json())
+            logger.info('获取数据', {'进度': num/len(date_list), '日期': date, '榜单类型': item_type})
             if not res.json().get('success', None):
-                print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
+                logger.error('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
                 continue
             if not res.json().get('data', {}).get('rows', None):
-                print("返回的数据字典异常, ['data']['rows'] 不能为空")
+                logger.error("返回的数据字典异常, ['data']['rows'] 不能为空")
                 num += 1
                 self.error_count += 1
                 time.sleep(1)
@@ -291,7 +290,7 @@ class AikuCun:
             num += 1
             time.sleep(1)
             if num % 32 == 0:
-                print("避免频繁请求, 正在休眠...")
+                logger.info("避免频繁请求, 正在休眠...")
                 # time.sleep(60)
 
         return results
@@ -413,18 +412,18 @@ class AikuCun:
             '尺码': 'varchar(50)',
             '货号': 'varchar(50)',  # 款号 + 颜色编码
         }
-        print(f'{self.shop_name} 正在更新数据库 {db_name} -> {table_name}...')
+        logger.info('更新数据库', {'店铺名称': self.shop_name, '库': db_name, '表': table_name})
         if 'spu' in table_name:
             drop_dup = ['日期', '平台', '店铺名称', '商品款号', '访客量']
         else:
             drop_dup = ['日期', '平台', '店铺名称', '条码']
-        uld.upload_data(
+        self.uld.upload_data(
             db_name=db_name,
             table_name=table_name,
             data=_results,
             set_typ=set_typ,  # 定义列和数据类型
             primary_keys=[],  # 创建唯一主键
-            check_duplicate=True,  # 检查重复数据
+            check_duplicate=False,  # 检查重复数据
             update_on_duplicate=False,  # 遇到重复时更新数据，默认 False 跳过
             duplicate_columns=drop_dup,  # 指定排重的组合键
             allow_null=False,  # 允许插入空值
@@ -470,36 +469,44 @@ class AikuCun:
             headers=headers,
             data=json.dumps(data)
         )
-        print(res.json())
 
 
 def main(start_date, end_date=None, item_type=['spu']):
-    ak = AikuCun()
-    # ak.get_sign()
-    for type_ in item_type:
-        if type_ not in ['spu', 'sku']:
-            print(f'{item_type} 非法参数: {type_}')
-            continue
-        for i in range(2):
-            data_list = ak.get_data_from_bbx(
-                start_date=start_date,
-                end_date=end_date,
-                item_type=type_,
-                page_num=1,
-                page_size=300
-            )
-            if not data_list:
-                ak.logining()
-                ak.save_token()
-                ak.error_count = 0  # 重置错误计数器
-            else:
-                break
+    db_config = {
+        'username': username,
+        'password': password,
+        'host': host,
+        'port': int(port),
+        'pool_size': 3
+    }
+    with uploader.MySQLUploader(**db_config) as uld:
+        with s_query.QueryDatas(**db_config) as download:
+            ak = AikuCun(uld_manager=uld, download_manager=download)
+            # ak.get_sign()
+            for type_ in item_type:
+                if type_ not in ['spu', 'sku']:
+                    logger.error(f'{item_type} 非法参数: {type_}')
+                    continue
+                for i in range(2):
+                    data_list = ak.get_data_from_bbx(
+                        start_date=start_date,
+                        end_date=end_date,
+                        item_type=type_,
+                        page_num=1,
+                        page_size=300
+                    )
+                    if not data_list:
+                        ak.logining()
+                        ak.save_token()
+                        ak.error_count = 0  # 重置错误计数器
+                    else:
+                        break
 
-        ak.insert_datas(
-            data_list=data_list,
-            db_name='爱库存2',
-            table_name=f'{type_}榜单'
-        )
+                ak.insert_datas(
+                    data_list=data_list,
+                    db_name='爱库存2',
+                    table_name=f'{type_}榜单'
+                )
 
 
 
@@ -508,7 +515,7 @@ if __name__ == '__main__':
         start_date='2025-05-13',
         # end_date='2025-04-28',  # 不传则默认到今天
         item_type=[
-            # 'spu',
+            'spu',
             'sku'
         ]
     )

{mdbq-4.0.9.dist-info → mdbq-4.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.9
+Version: 4.0.11
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.9.dist-info → mdbq-4.0.11.dist-info}/RECORD

@@ -1,18 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=N6Y-iQbP8CjtdAYEfFoeANx5OHaGKVkL-lm9Q2bnRnY,17
+mdbq/__version__.py,sha256=PQJs_Lgx6OvamcsXbLCVuBAvLc7j2xwJDZEWigwyUy8,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=SiUr-JC-L2YllQa6mTV8MaaY1LhATvHl4ApyBQdncrU,166695
+mdbq/aggregation/query_data.py,sha256=SM8cS9lBKmhLBQdwJz-sRu9bl7w1HS0MEq10s6Tqf_0,166777
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=9w_o5mYB3FooIxobq_lSa6oCYTKIhPxDFox-jeLtUHI,21714
-mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
+mdbq/mysql/deduplicator.py,sha256=fS1dSs92vN15tuqmAKrUVdKk6z9dwW_Fe9WHMBYsy2U,73172
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=FSFrFZE5yzEbnpLrN2AmlRZ_VvTvfpIWaQUjZfLIi9g,40342
+mdbq/mysql/s_query.py,sha256=RPC-KZVuqPlCSmpmtUmYAOJdxJT01i0DvlIbmum4MxM,42882
 mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=wNQE7UjCEyAKri9CnQXO7d6EVXCaYqFze2i2tcGAVpw,81001
+mdbq/mysql/uploader.py,sha256=wX2gHhVQJwGErnjUbLnsljkZ8Yd3YK-HS3P7q8DizAA,81053
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -24,8 +23,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.9.dist-info/METADATA,sha256=0yI54Kb8_qdyDNnmOWFaBmlgAONodPDBAYbaMKnAF5A,363
-mdbq-4.0.9.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.9.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=7oquQ2RIJr6B1xblQMfnmHzteOlvHA7dIcPRaAPfHBc,21546
+mdbq-4.0.11.dist-info/METADATA,sha256=zZh35aA-suJ3B_v39Mw8V_O2GSdOLdylfNPl_E99uqQ,364
+mdbq-4.0.11.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.11.dist-info/RECORD,,

mdbq/log/spider_logging.py

@@ -1,47 +0,0 @@
-import logging
-from logging.handlers import RotatingFileHandler
-import platform
-import os
-import sys
-import getpass
-
-
-def setup_logging(reMoveOldHandler=True, filename='spider_tg.log'):
-    """
-    reMoveOldHandler: 替换根日志记录器的所有现有处理器
-    """
-    dir_path = os.path.expanduser("~")
-    if not os.path.isdir(os.path.join(dir_path, 'logfile')):
-        os.makedirs(os.path.join(dir_path, 'logfile'))
-
-    log_file = os.path.join(dir_path, 'logfile', filename)
-    file_handler = RotatingFileHandler(
-        filename=log_file,
-        maxBytes=3*1024*1024,  # 3MB
-        backupCount=10,
-        encoding='utf-8'  # 明确指定编码（避免Windows乱码）
-    )
-    stream_handler = logging.StreamHandler()  # 终端输出Handler
-    formatter = logging.Formatter(
-        fmt='[%(asctime)s] %(levelname)s %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
-    )
-    file_handler.setFormatter(formatter)
-    stream_handler.setFormatter(formatter)  # 终端使用相同格式
-    file_handler.setLevel(logging.INFO)
-    stream_handler.setLevel(logging.INFO)
-
-    # 获取根日志记录器并添加Handler
-    logger = logging.getLogger()
-    if reMoveOldHandler:
-        # 移除根日志记录器的所有现有处理器
-        for handler in logger.handlers[:]:  # 使用[:]来创建handlers列表的一个副本，因为我们在迭代时修改列表
-            logger.removeHandler(handler)
-    logger.addHandler(file_handler)
-    logger.addHandler(stream_handler)
-    logger.setLevel(logging.INFO)  # 设置根日志级别
-    return logger
-
-
-if __name__ == '__main__':
-    pass