mdbq 4.0.9__tar.gz → 4.0.10__tar.gz

{mdbq-4.0.9 → mdbq-4.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.9
+Version: 4.0.10
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.10/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.0.10'

{mdbq-4.0.9 → mdbq-4.0.10}/mdbq/aggregation/query_data.py

@@ -1557,7 +1557,6 @@ class MysqlDatasQuery:
             'unique_keys': [['日期', '店铺id', '商品id']],  # 唯一约束列表
         }
 
-
     @upload_data_decorator()
     def spph(self, db_name='聚合数据', table_name='天猫_商品排行'):
         """  """
@@ -3685,11 +3684,12 @@ def main(months=3):
         password=password, 
         host=host, 
         port=port,
-        maxconnections=30,
+        maxconnections=20,
         )
     query1(download_manager=download_manager, months=months)
     query2(download_manager=download_manager, months=months)
     query3(download_manager=download_manager, months=months)
+    logger.info('数据聚合完成')
 
 
 if __name__ == '__main__':

{mdbq-4.0.9 → mdbq-4.0.10}/mdbq/mysql/s_query.py

@@ -47,9 +47,9 @@ class QueryDatas:
             host: 数据库主机
             port: 数据库端口
             charset: 字符集，默认utf8mb4
-            maxconnections: 最大连接数，默认20
-            mincached: 最小缓存连接数，默认2
-            maxcached: 最大缓存连接数，默认5
+            maxconnections: 最大活动连接数，默认20
+            mincached: 最小缓存连接数，空闲连接数量，默认2
+            maxcached: 最大缓存连接数，最大空闲连接数，默认5
             connect_timeout: 连接超时时间，默认10秒
             read_timeout: 读取超时时间，默认30秒
             write_timeout: 写入超时时间，默认30秒
@@ -253,20 +253,8 @@ class QueryDatas:
 
     # @_execute_with_retry
     def _get_connection(self, db_name: Optional[str] = None) -> pymysql.connections.Connection:
-        """
-        从连接池获取数据库连接
-        
-        Args:
-            db_name: 可选的数据库名，如果提供则会在连接后选择该数据库
-            
-        Returns:
-            数据库连接对象
-            
-        Raises:
-            ConnectionError: 当获取连接失败时抛出
-        """
+        """从连接池获取数据库连接"""
         try:
-            # 只在连续失败次数达到阈值时检查健康状态
             if self._pool_stats['consecutive_failures'] >= self._pool_stats['max_consecutive_failures']:
                 if not self._check_pool_health():
                     logger.warning('连接池不健康，尝试重新创建')
@@ -282,66 +270,184 @@ class QueryDatas:
             error_code = e.args[0] if e.args else None
             if error_code in (2003, 2006, 2013):
                 logger.error('数据库连接错误', {
+                    '库': db_name,
                     '错误代码': error_code,
                     '错误信息': str(e),
-                    '数据库': db_name
                 })
                 self.pool = self._create_connection_pool(10, 2, 5)
                 self._pool_stats['consecutive_failures'] = 0
                 raise ConnectionError(f'数据库连接错误: {str(e)}')
-            else:
-                raise
+            raise
         except Exception as e:
             logger.error('从连接池获取数据库连接失败', {
+                '库': db_name,
                 '错误': str(e),
-                '数据库': db_name
             })
             raise ConnectionError(f'连接数据库失败: {str(e)}')
 
     # @_execute_with_retry
-    def _execute_query(self, sql: str, params: tuple = None, db_name: str = None) -> Optional[List[Dict[str, Any]]]:
+    def _execute_query(self, sql: str, params: tuple = None, db_name: str = None, 
+                      fetch_all: bool = True, error_handling: bool = True) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
+        """执行SQL查询的通用方法"""
+        try:
+            if sql.upper().startswith('SHOW DATABASES'):
+                with closing(self._get_connection()) as connection:
+                    with closing(connection.cursor()) as cursor:
+                        cursor.execute(sql, params)
+                        return cursor.fetchall() if fetch_all else cursor.fetchone()
+            else:
+                with closing(self._get_connection(db_name)) as connection:
+                    with closing(connection.cursor()) as cursor:
+                        cursor.execute(sql, params)
+                        return cursor.fetchall() if fetch_all else cursor.fetchone()
+        except pymysql.OperationalError as e:
+            error_code = e.args[0] if e.args else None
+            if error_handling:
+                if error_code in (1045, 1049):  # 访问被拒绝或数据库不存在
+                    logger.error('数据库访问错误', {
+                        'SQL': sql,
+                        '参数': params,
+                        '库': db_name,
+                        '错误代码': error_code,
+                        '错误信息': str(e)
+                    })
+                else:
+                    logger.error('数据库操作错误', {
+                        '库': db_name,
+                        'SQL': sql,
+                        '参数': params,
+                        '错误代码': error_code,
+                        '错误信息': str(e)
+                    })
+                return None
+            raise
+        except Exception as e:
+            if error_handling:
+                logger.error('执行SQL查询失败', {
+                    '库': db_name,
+                    'SQL': sql,
+                    '参数': params,
+                    '错误类型': type(e).__name__,
+                    '错误信息': str(e)
+                })
+                return None
+            raise
+
+    def _get_table_info(self, db_name: str, table_name: str, info_type: Literal['columns', 'dtypes', 'exists'] = 'exists') -> Union[bool, List[Dict[str, Any]], List[str]]:
         """
-        执行SQL查询的通用方法。
+        获取表信息的通用方法。
         
         Args:
-            sql: SQL查询语句
-            params: 查询参数
             db_name: 数据库名
-            
+            table_name: 表名
+            info_type: 信息类型
+                - 'exists': 检查表是否存在（默认）
+                - 'columns': 获取列名列表
+                - 'dtypes': 获取列名和类型
+                
         Returns:
-            查询结果列表，如果查询失败返回None
+            根据info_type返回不同类型的信息：
+            - 'exists': 返回bool，表示表是否存在
+            - 'columns': 返回列名列表
+            - 'dtypes': 返回列名和类型的列表
         """
         try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    cursor.execute(sql, params)
-                    return cursor.fetchall()
+            if info_type == 'exists':
+                result = self._execute_query("SHOW DATABASES LIKE %s", (db_name,))
+                if not result:
+                    all_dbs = self._execute_query("SHOW DATABASES")
+                    available_dbs = [db['Database'] for db in all_dbs] if all_dbs else []
+                    logger.info('数据库不存在', {
+                        '库': db_name,
+                        '可用的数据库': available_dbs,
+                        '可能的原因': '数据库名称错误或没有访问权限'
+                    })
+                    return False
+                    
+                result = self._execute_query("SHOW TABLES LIKE %s", (table_name,), db_name=db_name)
+                if not result:
+                    all_tables = self._execute_query("SHOW TABLES", db_name=db_name)
+                    available_tables = [table[f'Tables_in_{db_name}'] for table in all_tables] if all_tables else []
+                    logger.info('表不存在', {
+                        '库': db_name,
+                        '表': table_name,
+                        '可用的表': available_tables,
+                        '可能的原因': '表名称错误或没有访问权限'
+                    })
+                    return False
+                return True
+                
+            elif info_type == 'columns':
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                result = self._execute_query(sql, (db_name, table_name))
+                return [col['COLUMN_NAME'] for col in result] if result else []
+                
+            elif info_type == 'dtypes':
+                sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                return self._execute_query(sql, (db_name, table_name)) or []
+                
         except Exception as e:
-            logger.error('执行SQL查询失败', {
-                'SQL': sql,
-                '参数': params,
-                '数据库': db_name,
+            logger.error('获取表信息失败', {
+                '库': db_name,
+                '表': table_name,
+                '信息类型': info_type,
                 '错误类型': type(e).__name__,
                 '错误信息': str(e)
             })
-            return None
+            return [] if info_type != 'exists' else False
 
-    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
-        """
-        按指定条件查询数据库表，返回满足条件的指定字段数据。
+    def check_infos(self, db_name: str, table_name: str) -> bool:
+        """检查数据库和数据表是否存在"""
+        return self._get_table_info(db_name, table_name, 'exists')
+
+    def _format_columns(self, columns: List[str]) -> str:
+        """格式化列名列表为SQL语句"""
+        return ', '.join([f'`{col}`' for col in columns])
+
+    def columns_to_list(self, db_name: str, table_name: str, columns_name: List[str], where: str = None) -> List[Dict[str, Any]]:
+        """获取数据表的指定列数据"""
+        if not self._get_table_info(db_name, table_name):
+            return []
         
-        Args:
-            db_name: 数据库名
-            table_name: 表名
-            condition: SQL条件字符串（不含WHERE）
-            columns: 查询字段字符串或以逗号分隔的字段名，默认'更新时间'
+        try:
+            existing_columns = self._get_table_info(db_name, table_name, 'columns')
+            columns_name = [col for col in columns_name if col in existing_columns]
             
-        Returns:
-            查询结果列表，如果查询失败返回None
-        """
-        if not self.check_infos(db_name, table_name):
-            return None
+            if not columns_name:
+                logger.info('未找到匹配的列名', {'库': db_name, '表': table_name, '请求列': columns_name})
+                return []
+            
+            sql = f"SELECT {self._format_columns(columns_name)} FROM `{db_name}`.`{table_name}`"
+            if where:
+                sql += f" WHERE {where}"
             
+            logger.debug('执行列查询', {'库': db_name, '表': table_name, 'SQL': sql})
+            return self._execute_query(sql, db_name=db_name) or []
+            
+        except Exception as e:
+            logger.error('列查询失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
+            return []
+
+    def dtypes_to_list(self, db_name: str, table_name: str, columns_name: List[str] = None) -> List[Dict[str, Any]]:
+        """获取数据表的列名和类型"""
+        if not self._get_table_info(db_name, table_name):
+            return []
+        
+        try:
+            result = self._get_table_info(db_name, table_name, 'dtypes')
+            if columns_name:
+                columns_name = set(columns_name)
+                result = [row for row in result if row['COLUMN_NAME'] in columns_name]
+            return result
+        except Exception as e:
+            logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
+            return []
+
+    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
+        """按指定条件查询数据库表"""
+        if not self._get_table_info(db_name, table_name):
+            return None
+        
         sql = f"SELECT {columns} FROM `{table_name}` WHERE {condition}"
         logger.debug('执行SQL查询', {'库': db_name, '表': table_name, 'SQL': sql})
         return self._execute_query(sql, db_name=db_name)
@@ -598,98 +704,6 @@ class QueryDatas:
                 df[col] = df[col].astype(float)
         return df
 
-    # @_execute_with_retry
-    def columns_to_list(self, db_name, table_name, columns_name, where: str = None) -> list:
-        """
-        获取数据表的指定列, 支持where条件筛选, 返回列表字典。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns_name: 需要获取的列名列表
-        :param where: 可选，SQL条件字符串（不含WHERE）
-        :return: [{列1:值, 列2:值, ...}, ...]
-        """
-        if not self.check_infos(db_name, table_name):
-            return []
-            
-        try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, table_name))
-                    cols_exist = [col['COLUMN_NAME'] for col in cursor.fetchall()]
-                    columns_name = [item for item in columns_name if item in cols_exist]
-                    if not columns_name:
-                        logger.info('未找到匹配的列名', {'库': db_name, '表': table_name, '请求列': columns_name})
-                        return []
-                    columns_in = ', '.join([f'`{col}`' for col in columns_name])
-                    sql = f"SELECT {columns_in} FROM `{db_name}`.`{table_name}`"
-                    if where:
-                        sql += f" WHERE {where}"
-                    logger.debug('执行列查询', {'库': db_name, '表': table_name, 'SQL': sql})
-                    cursor.execute(sql)
-                    column_values = cursor.fetchall()
-            return column_values
-        except Exception as e:
-            logger.error('列查询失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
-            return []
-
-    # @_execute_with_retry
-    def dtypes_to_list(self, db_name, table_name, columns_name=None) -> list:
-        """
-        获取数据表的列名和类型, 支持只返回部分字段类型。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns_name: 可选，字段名列表，仅返回这些字段的类型
-        :return: [{'COLUMN_NAME': ..., 'COLUMN_TYPE': ...}, ...]
-        """
-        if not self.check_infos(db_name, table_name):
-            return []
-            
-        try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, table_name))
-                    column_name_and_type = cursor.fetchall()
-                    if columns_name:
-                        columns_name = set(columns_name)
-                        column_name_and_type = [row for row in column_name_and_type if row['COLUMN_NAME'] in columns_name]
-            return column_name_and_type
-        except Exception as e:
-            logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
-            return []
-
-    # @_execute_with_retry
-    def check_infos(self, db_name, table_name) -> bool:
-        """
-        检查数据库和数据表是否存在。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :return: 存在返回True，否则False
-        """
-        try:
-            # 检查数据库是否存在
-            result = self._execute_query("SHOW DATABASES LIKE %s", (db_name,))
-            if not result:
-                logger.info('数据库不存在', {'库': db_name})
-                return False
-                
-            # 检查表是否存在
-            result = self._execute_query("SHOW TABLES LIKE %s", (table_name,), db_name=db_name)
-            if not result:
-                logger.info('表不存在', {'库': db_name, '表': table_name})
-                return False
-            return True
-            
-        except Exception as e:
-            logger.error('检查数据库或表失败', {
-                '库': db_name,
-                '表': table_name,
-                '错误类型': type(e).__name__,
-                '错误信息': str(e)
-            })
-            return False
-
     def __enter__(self):
         """上下文管理器入口"""
         return self
@@ -772,6 +786,8 @@ class QueryDatas:
             - 当return_format='list_dict'时，返回列表字典
             - 如果查询失败，返回空的DataFrame或空列表
         """
+        start_time = time.time()
+        
         if not db_name or not table_name:
             logger.error('数据库名和表名不能为空', {'库': db_name, '表': table_name})
             return [] if return_format == 'list_dict' else pd.DataFrame()
@@ -786,7 +802,7 @@ class QueryDatas:
         start_date, end_date = self._validate_date_range(start_date, end_date, db_name, table_name)
         
         # 检查数据库和表是否存在
-        if not self.check_infos(db_name, table_name):
+        if not self._get_table_info(db_name, table_name):
             return [] if return_format == 'list_dict' else pd.DataFrame()
         try:
             with closing(self._get_connection(db_name)) as connection:
@@ -863,7 +879,7 @@ class QueryDatas:
                     target_time = 1.0  # 期望每批1秒
 
                     while offset < total_count:
-                        start_time = time.time()
+                        _p_time = time.time()
                         # 添加分页参数
                         page_sql = f"{base_sql} LIMIT %s OFFSET %s"
                         page_params = list(params) + [page_size, offset]
@@ -881,7 +897,7 @@ class QueryDatas:
                             else:
                                 all_results = pd.concat([all_results, pd.DataFrame(page_results)], ignore_index=True)
 
-                        duration = time.time() - start_time
+                        duration = time.time() - _p_time
                         page_size = self._adjust_page_size(duration, page_size, min_size, max_size, target_time)
                         offset += len(page_results)
                         logger.debug('分页查询进度', {
@@ -896,6 +912,21 @@ class QueryDatas:
 
                     if return_format == 'df' and isinstance(all_results, pd.DataFrame) and not all_results.empty:
                         all_results = self._convert_decimal_columns(all_results)
+                    logger.info('查询完成', {
+                        '库': db_name,
+                        '表': table_name,
+                        '总记录数': total_count,
+                        '已获取记录数': len(all_results) if return_format == 'list_dict' else len(all_results.index),
+                        '查询耗时': f'{time.time() - start_time:.2f}s',
+                        '查询参数': {
+                            '开始日期': start_date,
+                            '结束日期': end_date,
+                            '日期字段': date_field,
+                            '限制行数': limit,
+                            '分页大小': page_size,
+                            '返回数据格式': return_format,
+                        }
+                    })
                     return all_results
 
         except Exception as e:
@@ -903,7 +934,15 @@ class QueryDatas:
                 '库': db_name,
                 '表': table_name,
                 '错误类型': type(e).__name__,
-                '错误信息': str(e)
+                '错误信息': str(e),
+                '查询参数': {
+                    '开始日期': start_date,
+                    '结束日期': end_date,
+                    '日期字段': date_field,
+                    '限制行数': limit,
+                    '分页大小': page_size,
+                    '返回数据格式': return_format,
+                }
             })
             return [] if return_format == 'list_dict' else pd.DataFrame()
 

{mdbq-4.0.9 → mdbq-4.0.10}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.9
+Version: 4.0.10
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.9/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '4.0.9'