mdbq 4.1.2__tar.gz → 4.1.3__tar.gz

{mdbq-4.1.2 → mdbq-4.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdbq
-Version: 4.1.2
+Version: 4.1.3
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.1.3/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.1.3'

{mdbq-4.1.2 → mdbq-4.1.3}/mdbq/myconf/myconf.py

@@ -1,22 +1,22 @@
 import re
 from typing import Dict, Any, Optional, Union, List, Tuple, Type, TypeVar
 from pathlib import Path
-from mdbq.log import mylogger
+# from mdbq.log import mylogger
 from dataclasses import dataclass, field
 from enum import Enum
 import time
 
-logger = mylogger.MyLogger(
-    logging_mode='both',
-    log_level='info',
-    log_format='json',
-    max_log_size=50,
-    backup_count=5,
-    enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段过滤
-    enable_metrics=False,  # 是否启用性能指标
-)
+# logger = mylogger.MyLogger(
+#     logging_mode='both',
+#     log_level='info',
+#     log_format='json',
+#     max_log_size=50,
+#     backup_count=5,
+#     enable_async=False,  # 是否启用异步日志
+#     sample_rate=1,  # 采样DEBUG/INFO日志
+#     sensitive_fields=[],  #  敏感字段过滤
+#     enable_metrics=False,  # 是否启用性能指标
+# )
 
 T = TypeVar('T')  # 类型变量
 
@@ -151,12 +151,10 @@ class ConfigParser:
     def open(self, file_path: Union[str, Path]) -> 'ConfigParser':
         """打开配置文件"""
         file_path = Path(file_path)
-        logger.debug(f'尝试打开配置文件: {file_path}')
         if not file_path.exists() and not self.options.auto_create:
-            logger.error(f'配置文件不存在: {file_path}')
+            # logger.error(f'配置文件不存在: {file_path}')
             raise ConfigException.file_not_found(file_path)
         self._current_file = file_path
-        logger.debug(f'配置文件已打开: {file_path}')
         return self
 
     def _ensure_file_open(self) -> None:
@@ -211,7 +209,6 @@ class ConfigParser:
     
     def _update_cache(self, file_path: str, config: Dict[str, Any]) -> None:
         """更新配置缓存"""
-        logger.debug(f'更新配置缓存: {file_path}')
         self._config_cache[file_path] = config
         self._cache_timestamps[file_path] = time.time()
     
@@ -236,13 +233,11 @@ class ConfigParser:
     def _clear_cache(self, file_path: Optional[str] = None) -> None:
         """清除配置缓存"""
         if file_path:
-            logger.debug(f'清除指定文件的缓存: {file_path}')
             self._config_cache.pop(file_path, None)
             self._cache_timestamps.pop(file_path, None)
             self._comments_cache.pop(file_path, None)
             self._section_map.pop(file_path, None)
         else:
-            logger.debug('清除所有配置缓存')
             self._config_cache.clear()
             self._cache_timestamps.clear()
             self._comments_cache.clear()
@@ -323,12 +318,11 @@ class ConfigParser:
                  default: Any = None, value_type: Optional[Type[T]] = None,
                  file_path: Optional[Union[str, Path]] = None) -> T:
         """获取指定配置项的值"""
-        logger.debug(f'获取配置项: section={section}, key={key}, file_path={file_path}')
         if file_path is None:
             self._ensure_file_open()
             file_path = self._current_file
         if not self._validate_key(key):
-            logger.error(f'无效的键名: {key}')
+            # logger.error(f'无效的键名: {key}')
             raise ConfigException.invalid_key_error(key, file_path, section)
         config = self.read(file_path)
         section = section or self.options.default_section
@@ -336,15 +330,15 @@ class ConfigParser:
         original_section = self._get_original_section(str(file_path), normalized_section)
         if original_section is None:
             if default is not None:
-                logger.warning(f'节不存在，返回默认值: section={section}, key={key}, default={default}')
+                # logger.warning(f'节不存在，返回默认值: section={section}, key={key}, default={default}')
                 return default
-            logger.error(f'配置节不存在: {section}')
+            # logger.error(f'配置节不存在: {section}')
             raise ConfigException.section_not_found(file_path, section)
         if key not in config[original_section]:
             if default is not None:
-                logger.warning(f'键不存在，返回默认值: section={section}, key={key}, default={default}')
+                # logger.warning(f'键不存在，返回默认值: section={section}, key={key}, default={default}')
                 return default
-            logger.error(f'配置键不存在: {key}')
+            # logger.error(f'配置键不存在: {key}')
             raise ConfigException.key_not_found(file_path, original_section, key)
         value = config[original_section][key]
         if value_type is not None:
@@ -356,7 +350,6 @@ class ConfigParser:
                   value_types: Optional[Dict[str, Type]] = None,
                   file_path: Optional[Union[str, Path]] = None) -> Dict[str, Any]:
         """批量获取多个配置项的值"""
-        logger.debug(f'批量获取配置项: section={section}, keys={keys}, file_path={file_path}')
         if file_path is None:
             self._ensure_file_open()
             file_path = self._current_file
@@ -375,7 +368,7 @@ class ConfigParser:
                 )
                 result[key] = value
             except ConfigException as e:
-                logger.error(f"读取配置项失败: section={section}, key={key}, error={e}")
+                # logger.error(f"读取配置项失败: section={section}, key={key}, error={e}")
                 if key in defaults:
                     result[key] = defaults[key]
                 else:
@@ -388,7 +381,6 @@ class ConfigParser:
                           value_types: Optional[Dict[str, Type]] = None,
                           file_path: Optional[Union[str, Path]] = None) -> Tuple[Any, ...]:
         """获取指定节点下多个键的值元组"""
-        logger.debug(f'获取节下多个键的值: section={section}, keys={keys}, file_path={file_path}')
         if file_path is None:
             self._ensure_file_open()
             file_path = self._current_file
@@ -407,7 +399,7 @@ class ConfigParser:
                 )
                 result.append(value)
             except ConfigException as e:
-                logger.error(f"读取配置项失败: section={section}, key={key}, error={e}")
+                # logger.error(f"读取配置项失败: section={section}, key={key}, error={e}")
                 if key in defaults:
                     result.append(defaults[key])
                 else:
@@ -419,7 +411,6 @@ class ConfigParser:
                  value_type: Optional[Type] = None,
                  file_path: Optional[Union[str, Path]] = None) -> None:
         """设置指定配置项的值"""
-        logger.debug(f'准备写入配置项: section={section}, key={key}, value={value}, file_path={file_path}')
         if file_path is None:
             self._ensure_file_open()
             file_path = self._current_file
@@ -427,7 +418,7 @@ class ConfigParser:
             file_path = Path(file_path)
             
         if not self._validate_key(key):
-            logger.error(f'无效的键名: {key}')
+            # logger.error(f'无效的键名: {key}')
             raise ConfigException.invalid_key_error(key, file_path, section)
             
         section = section or self.options.default_section
@@ -452,7 +443,7 @@ class ConfigParser:
                     else:
                         value = value_type(value)
                 except (ValueError, TypeError) as e:
-                    logger.error(f'类型转换失败: value={value}, type={value_type}, error={e}')
+                    # logger.error(f'类型转换失败: value={value}, type={value_type}, error={e}')
                     raise ConfigException.conversion_error(value, value_type, file_path, section=section, key=key)
                     
             if isinstance(value, bool):
@@ -509,21 +500,20 @@ class ConfigParser:
                     file.write(f'{key}={value}\n')
                     
             self._clear_cache(str(file_path))
-            logger.info(f'配置项写入成功: section={section}, key={key}, value={value}, file_path={file_path}')
+            # logger.info(f'配置项写入成功: section={section}, key={key}, value={value}, file_path={file_path}')
             
         except Exception as e:
-            logger.error(f'写入配置项失败: section={section}, key={key}, value={value}, file_path={file_path}, error={e}')
+            # logger.error(f'写入配置项失败: section={section}, key={key}, value={value}, file_path={file_path}, error={e}')
             raise ConfigException.write_error(file_path, e)
 
     def set_values(self, section: Optional[str] = None, values: Dict[str, Any] = None,
                   value_types: Optional[Dict[str, Type]] = None,
                   file_path: Optional[Union[str, Path]] = None) -> None:
         """批量设置多个配置项的值"""
-        logger.debug(f'批量写入配置项: section={section}, keys={list(values.keys())}, file_path={file_path}')
         for key, value in values.items():
             value_type = value_types.get(key) if value_types else None
             self.set_value(section, key, value, value_type, file_path)
-        logger.info(f'批量写入配置项完成: section={section}, file_path={file_path}')
+        # logger.info(f'批量写入配置项完成: section={section}, file_path={file_path}')
 
     def validate_config(self, section: Optional[str] = None, schema: Dict[str, Type] = None,
                        file_path: Optional[Union[str, Path]] = None) -> bool:
@@ -560,17 +550,15 @@ class ConfigParser:
         else:
             file_path = Path(file_path)
         
-        logger.debug(f'开始读取配置文件: {file_path}')
         cached_config = self._get_cached_config(str(file_path))
         if cached_config is not None:
-            logger.debug(f'命中配置缓存: {file_path}')
             return cached_config
         
         if not file_path.exists():
             if not self.options.auto_create:
-                logger.error(f'配置文件不存在: {file_path}')
+                # logger.error(f'配置文件不存在: {file_path}')
                 raise ConfigException.file_not_found(file_path)
-            logger.info(f'配置文件不存在，将创建: {file_path}')
+            # logger.info(f'配置文件不存在，将创建: {file_path}')
             file_path.parent.mkdir(parents=True, exist_ok=True)
             file_path.touch()
             return {}
@@ -592,7 +580,7 @@ class ConfigParser:
                     if stripped_line.startswith('[') and stripped_line.endswith(']'):
                         current_section = stripped_line[1:-1]
                         if not self._validate_key(current_section):
-                            logger.error(f'无效的节名: {current_section}')
+                            # logger.error(f'无效的节名: {current_section}')
                             raise ConfigException.invalid_section_error(current_section, file_path)
                         self._update_section_map(str(file_path), current_section)
                         if current_section not in config:
@@ -606,7 +594,7 @@ class ConfigParser:
                     if key_value:
                         key, value = key_value
                         if not self._validate_key(key):
-                            logger.error(f'无效的键名: {key}')
+                            # logger.error(f'无效的键名: {key}')
                             raise ConfigException.invalid_key_error(key, file_path, current_section)
                         value, comment = self._extract_comment(value)
                         
@@ -621,11 +609,10 @@ class ConfigParser:
                             self._comments_cache.setdefault(str(file_path), {}).setdefault(current_section, []).append(comment)
                 
                 self._update_cache(str(file_path), config)
-                logger.debug(f'配置文件读取成功: {file_path}')
                 return config
                 
         except Exception as e:
-            logger.error(f'读取配置文件失败: {file_path}, error={e}')
+            # logger.error(f'读取配置文件失败: {file_path}, error={e}')
             raise ConfigException.read_error(file_path, e)
 
 

{mdbq-4.1.2 → mdbq-4.1.3}/mdbq/mysql/s_query.py

@@ -16,7 +16,7 @@ from functools import wraps
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
     logging_mode='file',
-    log_level='info',
+    log_level='error',
     log_format='json',
     max_log_size=50,
     backup_count=5,

{mdbq-4.1.2 → mdbq-4.1.3}/mdbq/redis/getredis.py

@@ -5,21 +5,21 @@ import numpy as np
 import json
 import datetime
 import threading
-from mdbq.log import mylogger
+# from mdbq.log import mylogger
 from decimal import Decimal
 import orjson
 
-logger = mylogger.MyLogger(
-    logging_mode='file',
-    log_level='info',
-    log_format='json',
-    max_log_size=50,
-    backup_count=5,
-    enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段过滤
-    enable_metrics=False,  # 是否启用性能指标
-)
+# logger = mylogger.MyLogger(
+#     logging_mode='file',
+#     log_level='info',
+#     log_format='json',
+#     max_log_size=50,
+#     backup_count=5,
+#     enable_async=False,  # 是否启用异步日志
+#     sample_rate=1,  # 采样DEBUG/INFO日志
+#     sensitive_fields=[],  #  敏感字段过滤
+#     enable_metrics=False,  # 是否启用性能指标
+# )
 
 
 class RedisData(object):
@@ -61,7 +61,8 @@ class RedisData(object):
 
         combined_df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
         if combined_df.empty:
-            logger.info(f"警告: {db_name}.{table_name} 未读取到数据")
+            # logger.info(f"警告: {db_name}.{table_name} 未读取到数据")
+            pass
         else:
             combined_df = self._convert_date_columns(combined_df)
         return combined_df
@@ -86,7 +87,7 @@ class RedisData(object):
             ttl = self.redis_engine.ttl(cache_key)
             cache_data = self._fetch_redis_data(cache_key)
         except Exception as e:
-            logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
+            # logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
             return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
 
         # 缓存失效处理逻辑
@@ -134,11 +135,11 @@ class RedisData(object):
             self.redis_engine.set(cache_key, serialized_data)
             self.redis_engine.expire(cache_key, self.cache_ttl)
 
-            logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
+            # logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
             return combined_data
 
         except Exception as e:
-            logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
+            # logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
             return pd.DataFrame()
 
     # Helper Methods ------------------------------------------------
@@ -160,7 +161,7 @@ class RedisData(object):
                 projection={}
             )
         except Exception as e:
-            logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
+            # logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
             return pd.DataFrame()
 
     def _fetch_redis_data(self, cache_key: str) -> pd.DataFrame:
@@ -173,7 +174,7 @@ class RedisData(object):
             df = pd.DataFrame(json.loads(data.decode("utf-8")))
             return self._convert_date_columns(df)
         except Exception as e:
-            logger.error(f"Redis 数据解析失败 {cache_key}: {e}")
+            # logger.error(f"Redis 数据解析失败 {cache_key}: {e}")
             return pd.DataFrame()
 
     def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:
@@ -278,7 +279,8 @@ class RedisDataHash(object):
 
         combined_df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
         if combined_df.empty:
-            logger.warn(f"warning: {db_name}.{table_name} 未读取到数据")
+            # logger.warn(f"warning: {db_name}.{table_name} 未读取到数据")
+            pass
         else:
             combined_df = self._convert_date_columns(combined_df)
         return combined_df
@@ -293,7 +295,7 @@ class RedisDataHash(object):
             projection={}
     ) -> pd.DataFrame:
         if not self.redis_engine.ping():
-            logger.error(f"Redis ping异常，直接访问 MySQL")
+            # logger.error(f"Redis ping异常，直接访问 MySQL")
             return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date, projection)
         start_dt = pd.to_datetime(start_date).floor('D')
         end_dt = pd.to_datetime(end_date).floor('D')
@@ -335,7 +337,7 @@ class RedisDataHash(object):
             return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date, projection)
 
         except Exception as e:
-            logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
+            # logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
             return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date, projection)
 
     def set_redis(
@@ -385,9 +387,10 @@ class RedisDataHash(object):
                         serialized_data = self._serialize_data(group)
                         self.redis_engine.hset(cache_key, month_str, serialized_data)
                     self.redis_engine.expire(cache_key, self.cache_ttl + random.randint(0, 1800))
-                logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
+                # logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
         except Exception as e:
-            logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
+            # logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
+            pass
 
     def _fetch_table_data(
             self,
@@ -406,7 +409,7 @@ class RedisDataHash(object):
                 projection=projection
             )
         except Exception as e:
-            logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
+            # logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
             return pd.DataFrame()
 
     def _fetch_redis_data(self, cache_key: str, months: list = None) -> pd.DataFrame:
@@ -441,7 +444,8 @@ class RedisDataHash(object):
                             df = self._convert_date_columns(df)
                             dfs.append(df)
                         except Exception as e:
-                            logger.error(f"月份数据解析失败 {field}: {e}")
+                            # logger.error(f"月份数据解析失败 {field}: {e}")
+                            pass
 
             # 处理分片数据（优化后的批处理逻辑）
             cursor, shard_data = results[result_index]
@@ -459,7 +463,8 @@ class RedisDataHash(object):
                             df = pd.DataFrame(orjson.loads(value))
                             dfs.append(self._convert_date_columns(df))
                         except Exception as e:
-                            logger.error(f"分片数据解析失败: {e}")
+                            # logger.error(f"分片数据解析失败: {e}")
+                            pass
 
                 # 继续获取后续分片
                 if cursor == 0:
@@ -475,7 +480,7 @@ class RedisDataHash(object):
             return pd.DataFrame()
 
         except Exception as e:
-            logger.error(f"Redis 数据获取失败 {cache_key}: {e}")
+            # logger.error(f"Redis 数据获取失败 {cache_key}: {e}")
             return pd.DataFrame()
 
     def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:
@@ -562,7 +567,7 @@ class RedisDataHash(object):
         try:
             records = temp_df.to_dict(orient='records')
         except Exception as e:
-            logger.error(f"DataFrame转字典失败: {str(e)}")
+            # logger.error(f"DataFrame转字典失败: {str(e)}")
             records = []
 
         # 序列化配置 --------------------------------------------------------