mdbq 3.12.7__py3-none-any.whl → 4.0.0__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.12.7'
+VERSION = '4.0.0'

mdbq/log/mylogger.py

@@ -18,6 +18,14 @@ try:
 except ImportError:
     HAS_PSUTIL = False
 
+def get_caller_filename(default='mylogger'):
+    stack = inspect.stack()
+    for frame_info in stack:
+        filename = frame_info.filename
+        # 跳过本日志库自身
+        if not filename.endswith('mylogger.py'):
+            return os.path.splitext(os.path.basename(filename))[0]
+    return default
 
 class MyLogger:
     """
@@ -52,10 +60,10 @@ class MyLogger:
 
     def __init__(
             self,
-            name: str = 'mylogger.log',
+            name: Optional[str] = None,
             logging_mode: str = 'console',  # 'both', 'console', 'file', 'none'
             log_level: str = 'INFO',  # 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'
-            log_file: str = 'm_app.log',
+            log_file: Optional[str] = None,
             log_format: str = 'json',  # 默认json格式，可选'simple'
             max_log_size: int = 50,  # MB
             backup_count: int = 5,
@@ -89,10 +97,16 @@ class MyLogger:
         :param flush_interval: 定时刷新日志器间隔(秒)
         """
         log_path = os.path.join(os.path.expanduser("~"), 'logfile')
+        if name is None:
+            name = get_caller_filename()
         self.name = name
         self.logging_mode = logging_mode.lower()
         self.log_level = log_level.upper()
-        self.log_file = os.path.join(log_path, log_file)
+        # log_file 自动为 name+'.log'
+        if log_file is None:
+            self.log_file = os.path.join(log_path, f"{self.name}.log")
+        else:
+            self.log_file = os.path.join(log_path, log_file)
         if not os.path.isdir(os.path.dirname(self.log_file)):
             os.makedirs(os.path.dirname(self.log_file))
         self.log_format = log_format
@@ -116,22 +130,17 @@ class MyLogger:
         self._last_metrics_time = 0
         self._metrics_cache = {}
 
-        # 异步日志相关
-        self._log_queue = queue.Queue(maxsize=buffer_size)
-        self._async_thread = None
-        self._stop_event = threading.Event()
-
-        # 定时刷新相关
-        self._flush_thread = None
-        self._last_flush_time = 0
-        self._start_flush_thread()
+        # 异步日志相关（标准库实现）
+        self._log_queue = None
+        self._queue_listener = None
+        self._handlers = []
 
         # 创建日志记录器
         self.logger = logging.getLogger(name)
         self._init_logging()
 
         if self.enable_async:
-            self._start_async_logging()
+            self._setup_async_logging()
 
         atexit.register(self.shutdown)
 
@@ -186,59 +195,37 @@ class MyLogger:
 
         # 定义日志格式
         if self.log_format.lower() == 'simple':
-            # 简单文本格式
             class SimpleFormatter(logging.Formatter):
                 def format(self, record):
-                    # 基础日志信息
                     msg = super().format(record)
-
-                    # 添加上下文信息
                     if hasattr(record, 'extra_data') and record.extra_data:
                         context_data = record.extra_data.get('context_data', {})
                         if context_data:
                             msg += f" | Context: {context_data}"
-
-                        # 添加性能指标
                         metrics = record.extra_data.get('性能指标', {})
                         if metrics:
                             msg += f" | Metrics: {metrics}"
-
-                        # 添加其他额外数据
                         extra = {k: v for k, v in record.extra_data.items()
                                  if k not in ('context_data', '性能指标')}
                         if extra:
                             msg += f" | Extra: {extra}"
-
                     return msg
-
             formatter = SimpleFormatter('%(asctime)s - %(levelname)s - %(message)s')
             formatter.datefmt = '%Y-%m-%d %H:%M:%S'
         else:
-            # 结构化JSON格式
             class StructuredFormatter(logging.Formatter):
                 def format(self, record):
                     log_data = {
-                        'timestamp': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                        'time': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                         'level': record.levelname,
-                        'message': record.getMessage(),
-                        # 'name': record.name,
-                        # 'module': record.module,
-                        # 'function': record.funcName,
+                        'msg': record.getMessage(),
                     }
-
-                    # 添加额外字段
                     if hasattr(record, 'extra_data'):
                         log_data.update(record.extra_data)
-
-                    # 添加上下文信息
                     if hasattr(record, 'context_data'):
                         log_data.update(record.context_data)
-
-                    # 添加异常信息
                     if record.exc_info:
                         log_data['异常'] = self.formatException(record.exc_info)
-
-                    # 过滤敏感信息
                     if hasattr(record, 'extra_data') and '过滤' in record.extra_data:
                         sensitive_fields = record.extra_data['过滤']
                         for field in sensitive_fields:
@@ -246,23 +233,19 @@ class MyLogger:
                                 log_data[field] = '***'
                             if isinstance(log_data.get('message'), str):
                                 log_data['message'] = log_data['message'].replace(field, '***')
-
                     return json.dumps(log_data, ensure_ascii=False, default=str)
-
             formatter = StructuredFormatter()
 
-        # 根据模式添加相应的handler
+        # 只创建handlers，不加到logger上（异步时由QueueListener管理）
+        self._handlers = []
         if self.logging_mode in ('both', 'console'):
             console_handler = logging.StreamHandler()
             console_handler.setFormatter(formatter)
-            self.logger.addHandler(console_handler)
-
+            self._handlers.append(console_handler)
         if self.logging_mode in ('both', 'file'):
-            # 确保日志目录存在
             log_dir = os.path.dirname(self.log_file)
             if log_dir and not os.path.exists(log_dir):
                 os.makedirs(log_dir)
-
             file_handler = logging.handlers.RotatingFileHandler(
                 filename=self.log_file,
                 maxBytes=self.max_log_size * 1024 * 1024,
@@ -271,7 +254,21 @@ class MyLogger:
                 delay=False
             )
             file_handler.setFormatter(formatter)
-            self.logger.addHandler(file_handler)
+            self._handlers.append(file_handler)
+
+        if not self.enable_async:
+            for handler in self._handlers:
+                self.logger.addHandler(handler)
+
+    def _setup_async_logging(self):
+        # 标准库异步日志实现
+        self._log_queue = queue.Queue(maxsize=self.buffer_size)  # 无限长度，绝不阻塞
+        queue_handler = logging.handlers.QueueHandler(self._log_queue)
+        self.logger.addHandler(queue_handler)
+        self._queue_listener = logging.handlers.QueueListener(
+            self._log_queue, *self._handlers, respect_handler_level=True
+        )
+        self._queue_listener.start()
 
     def _get_system_metrics(self) -> Dict[str, Any]:
         """获取系统资源使用指标"""
@@ -319,34 +316,6 @@ class MyLogger:
                                     extra={'extra_data': {'filter_error': str(e)}})
         return True  # 所有过滤器都返回 True，则记录该日志
 
-    def _async_log_worker(self):
-        """异步日志工作线程"""
-        while not self._stop_event.is_set() or not self._log_queue.empty():
-            try:
-                log_args = self._log_queue.get(timeout=0.1)
-                if log_args:
-                    level, message, extra = log_args
-                    self._sync_log(level, message, extra)
-            except queue.Empty:
-                continue
-            except Exception as e:
-                # 防止日志线程崩溃
-                try:
-                    self.logger.error(f"日志线程崩溃: {e}",
-                                      extra={'extra_data': {'async_error': str(e)}})
-                except:
-                    pass
-
-    def _start_async_logging(self):
-        """启动异步日志线程"""
-        self._stop_event.clear()
-        self._async_thread = threading.Thread(
-            target=self._async_log_worker,
-            name=f"{self.name}_async_logger",
-            daemon=True
-        )
-        self._async_thread.start()
-
     def log_error_handler(retry_times=0, fallback_level='error'):
         """
         日志错误处理装饰器
@@ -387,7 +356,7 @@ class MyLogger:
 
     @log_error_handler(retry_times=1, fallback_level='warning')
     def _sync_log(self, level: str, message: str, extra: Optional[Dict] = None):
-        """同步日志记录"""
+        """同步日志记录（兼容异步，直接走logger）"""
         if not hasattr(self.logger, level.lower()):
             return
 
@@ -430,7 +399,7 @@ class MyLogger:
         if not self._apply_filters(level, message, log_extra):
             return
 
-        # 记录日志
+        # 记录日志（直接走logger，异步/同步由handler决定）
         getattr(self.logger, level.lower())(message, extra={'extra_data': log_extra})
 
     def log(self, level: str, message: str, extra: Optional[Dict] = None):
@@ -443,18 +412,7 @@ class MyLogger:
         """
         if not hasattr(self.logger, level.lower()):
             return
-
-        if self.enable_async:
-            try:
-                self._log_queue.put((level, message, extra), timeout=0.1)
-            except queue.Full:
-                # 队列满时降级为同步日志，添加队列状态信息到extra
-                if extra is None:
-                    extra = {}
-                extra['queue_status'] = 'full'
-                self._sync_log(level, message, extra)
-        else:
-            self._sync_log(level, message, extra)
+        self._sync_log(level, message, extra)
 
     def set_level(self, level: str):
         """动态设置日志级别"""
@@ -596,26 +554,19 @@ class MyLogger:
 
     def shutdown(self):
         """关闭日志记录器，确保所有日志被刷新"""
-        if self.enable_async:
-            self._stop_event.set()
-            # 等待队列清空
-            while not self._log_queue.empty():
-                time.sleep(0.1)
-            if self._async_thread and self._async_thread.is_alive():
-                self._async_thread.join(timeout=0.5)
-
-        # 确保所有handler被刷新
-        if self._flush_thread:
-            self._flush_handlers()
-            if self._flush_thread.is_alive():
-                self._flush_thread.join(timeout=0.5)
-
+        if self.enable_async and self._queue_listener:
+            self._queue_listener.stop()
         # 关闭所有handler
         for handler in self.logger.handlers:
             try:
                 handler.close()
             except:
                 pass
+        for handler in getattr(self, '_handlers', []):
+            try:
+                handler.close()
+            except:
+                pass
 
 def main():
     # 创建日志记录器

{mdbq-3.12.7.dist-info → mdbq-4.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.12.7
+Version: 4.0.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.12.7.dist-info → mdbq-4.0.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=R4aIh4bYjUTB7AEjcPn5iJfp5jMY5s481CDtN5JhL_k,18
+mdbq/__version__.py,sha256=rGzKotWztr8AKlJYeZkowEC43bIAfiOqR7lVganyrJE,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=Crw6LwVo3I3IUbzIETu8f46Quza3CTCh-qYf4edbBPo,24139
+mdbq/log/mylogger.py,sha256=qBOHJK_h6R_SpfQ1yC5fAlJIEm6uro810i-47uA9C_U,22872
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=iMloLJz4i0w5UGyIwya-GiQS1iMq-DMmA7XPU8PP5k4,73138
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=GaekqY55pDEgVxbeQzHHshnQMC2YDv3v4mA7cQwjli4,21019
-mdbq-3.12.7.dist-info/METADATA,sha256=-8Uu-vIm1_S6-2fqFV9AnzLDD9xnDBySgXXqZKDPIqY,364
-mdbq-3.12.7.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.12.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.12.7.dist-info/RECORD,,
+mdbq-4.0.0.dist-info/METADATA,sha256=oGG1nyu37HUrUS5Tes3s5NeleIiE_neE2-rCvcceMaU,363
+mdbq-4.0.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.0.dist-info/RECORD,,