mdbq 3.9.6__py3-none-any.whl → 3.9.8__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.9.6'
+VERSION = '3.9.8'

mdbq/log/mylogger.py

@@ -9,6 +9,8 @@ import threading
 import queue
 from typing import Optional, Dict, Any, List, Callable, Union
 import atexit
+import traceback
+import inspect
 
 try:
     import psutil
@@ -65,7 +67,8 @@ class MyLogger:
             filters: Optional[List[Callable]] = None,
             enable_metrics: bool = False,
             metrics_interval: int = 300,
-            message_limited: int = 1000
+            message_limited: int = 1000,
+            flush_interval: int = 5
     ):
         """
         初始化日志记录器
@@ -84,15 +87,15 @@ class MyLogger:
         :param enable_metrics: 是否启用系统指标采集
         :param metrics_interval: 指标采集间隔(秒)
         :param message_limited: 简化日志内容，避免过长
+        :param flush_interval: 定时刷新日志器间隔(秒)
         """
         log_path = os.path.join(os.path.expanduser("~"), 'logfile')
-        if not os.path.isdir(log_path):
-            os.makedirs(log_path)
-
         self.name = name
         self.logging_mode = logging_mode.lower()
         self.log_level = log_level.upper()
         self.log_file = os.path.join(log_path, log_file)
+        if not os.path.isdir(os.path.dirname(self.log_file)):
+            os.makedirs(os.path.dirname(self.log_file))
         self.log_format = log_format
         self.max_log_size = max_log_size
         self.backup_count = backup_count
@@ -104,6 +107,7 @@ class MyLogger:
         self.enable_metrics = enable_metrics and HAS_PSUTIL
         self.metrics_interval = metrics_interval
         self.message_limited = message_limited
+        self.flush_interval = flush_interval
 
         # 上下文相关
         self._context = threading.local()
@@ -118,6 +122,11 @@ class MyLogger:
         self._async_thread = None
         self._stop_event = threading.Event()
 
+        # 定时刷新相关
+        self._flush_thread = None
+        self._last_flush_time = 0
+        self._start_flush_thread()
+
         # 创建日志记录器
         self.logger = logging.getLogger(name)
         self._init_logging()
@@ -213,12 +222,9 @@ class MyLogger:
                         'timestamp': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                         'level': record.levelname,
                         'message': record.getMessage(),
-                        # 'module': record.module,
-                        'function': record.funcName,
-                        # 'line': record.lineno,
-                        # 'thread': record.threadName,
-                        # 'process': record.processName,
                         'name': record.name,
+                        # 'module': record.module,
+                        # 'function': record.funcName,
                     }
 
                     # 添加额外字段
@@ -341,6 +347,45 @@ class MyLogger:
         )
         self._async_thread.start()
 
+    def log_error_handler(retry_times=0, fallback_level='error'):
+        """
+        日志错误处理装饰器
+
+        参数:
+        - retry_times: 异常时重试次数
+        - fallback_level: 降级日志级别
+        """
+
+        def decorator(log_method):
+            def wrapper(self, level: str, message: str, extra: Optional[Dict] = None):
+                last_exception = None
+                for attempt in range(retry_times + 1):
+                    try:
+                        return log_method(self, level, message, extra)
+                    except Exception as e:
+                        last_exception = e
+                        if attempt < retry_times:
+                            time.sleep(0.1 * (attempt + 1))  # 简单的退避策略
+                            continue
+
+                        try:
+                            # 降级处理
+                            logging.basicConfig()
+                            fallback_logger = logging.getLogger(f"{getattr(self, 'name', 'mylogger')}_fallback")
+                            fallback_msg = f"[降级处理] {message}"[:1000]
+                            getattr(fallback_logger, fallback_level)(
+                                f"日志记录失败(尝试{attempt + 1}次): {e}\n原始消息: {fallback_msg}"
+                            )
+                        except:
+                            sys.stderr.write(f"严重: 日志系统完全失败 - {last_exception}\n")
+
+                return None
+
+            return wrapper
+
+        return decorator
+
+    @log_error_handler(retry_times=1, fallback_level='warning')
     def _sync_log(self, level: str, message: str, extra: Optional[Dict] = None):
         """同步日志记录"""
         if not hasattr(self.logger, level.lower()):
@@ -369,7 +414,8 @@ class MyLogger:
             log_extra['context_data'] = self._context.data.copy()
 
         # 添加敏感字段过滤
-        log_extra['过滤'] = self.sensitive_fields
+        if self.sensitive_fields:
+            log_extra['过滤'] = self.sensitive_fields
 
         # 应用日志采样
         if self.sample_rate < 1.0 and level.lower() in ('debug', 'info'):
@@ -435,18 +481,6 @@ class MyLogger:
         if hasattr(self._context, 'data'):
             self._context.data.clear()
 
-    def shutdown(self):
-        """关闭日志记录器，确保所有日志被刷新"""
-        if self.enable_async:
-            self._stop_event.set()
-            if self._async_thread and self._async_thread.is_alive():
-                self._async_thread.join(timeout=5)
-
-        # 确保所有handler被刷新
-        for handler in self.logger.handlers:
-            handler.flush()
-            handler.close()
-
     def debug(self, message: str, extra: Optional[Dict] = None):
         """记录调试信息"""
         self.log('debug', message, extra)
@@ -471,41 +505,46 @@ class MyLogger:
         """记录异常信息"""
         if not extra:
             extra = {}
-        # # 获取异常发生的实际位置
+
+        # # 获取完整的异常堆栈
         # tb = exc_info.__traceback__
+        # while tb.tb_next:
+        #     tb = tb.tb_next  # 获取最内层的堆栈帧
         #
-        # if tb:
-        #     extra.update({
-        #         'module': tb.tb_frame.f_globals.get('__name__', ''),
-        #         'function': tb.tb_frame.f_code.co_name,
-        #         'line': tb.tb_lineno,
-        #         'file': tb.tb_frame.f_code.co_filename
-        #     })
-        # extra['异常'] = str(exc_info)
-        # extra['类型'] = exc_info.__class__.__name__
-        # self.log('error', message, extra)
-
-        # 获取完整的异常堆栈
-        tb = exc_info.__traceback__
-        while tb.tb_next:
-            tb = tb.tb_next  # 获取最内层的堆栈帧
-
-        extra.update({
-            'module': tb.tb_frame.f_globals.get('__name__', ''),
-            'function': tb.tb_frame.f_code.co_name,
-            'line': tb.tb_lineno,
-            'file': tb.tb_frame.f_code.co_filename,
-            '异常': str(exc_info),
-            '类型': exc_info.__class__.__name__,
-            '堆栈': self._format_traceback(exc_info)
-        })
+        # extra.update({
+        #     'module': tb.tb_frame.f_globals.get('__name__', ''),
+        #     'function': tb.tb_frame.f_code.co_name,
+        #     'line': tb.tb_lineno,
+        #     'file': tb.tb_frame.f_code.co_filename,
+        #     '异常': str(exc_info),
+        #     '类型': exc_info.__class__.__name__,
+        #     '堆栈': self._format_traceback(exc_info)
+        # })
+
+        # 使用inspect获取调用栈
+        frame = inspect.currentframe()
+        try:
+            # 向上追溯2层（1层是exception方法本身，2层是实际调用位置）
+            caller_frame = frame.f_back.f_back
+            extra.update({
+                'module': caller_frame.f_globals.get('__name__', ''),
+                'function': caller_frame.f_code.co_name,
+                'line': caller_frame.f_lineno,
+                'file': caller_frame.f_code.co_filename,
+                '异常': str(exc_info),
+                '类型': exc_info.__class__.__name__,
+                '堆栈': self._format_traceback(exc_info)
+            })
+        finally:
+            del frame  # 避免循环引用
 
         # 直接使用logger的error方法记录，保留原始调用栈
         self.log('error', message, extra)
 
     def _format_traceback(self, exc_info):
         """格式化异常堆栈"""
-        import traceback
+        if exc_info is None:
+            return ""
         return ''.join(traceback.format_exception(type(exc_info), exc_info, exc_info.__traceback__))
 
     def timeit(self, message: str = "Execution time"):
@@ -528,6 +567,63 @@ class MyLogger:
                              extra={'elapsed_seconds': elapsed})
             return False
 
+    def _start_flush_thread(self):
+        """启动定时刷新线程"""
+        self._stop_event.clear()
+        self._flush_thread = threading.Thread(
+            target=self._flush_worker,
+            name=f"{self.name}_flush_thread",
+            daemon=True
+        )
+        self._flush_thread.start()
+
+    def _flush_worker(self):
+        """定时刷新工作线程"""
+        while not self._stop_event.is_set():
+            try:
+                time.sleep(self.flush_interval)
+                self._flush_handlers()
+            except Exception as e:
+                try:
+                    self.logger.error(f"刷新线程异常: {e}",
+                                      extra={'extra_data': {'flush_error': str(e)}})
+                except:
+                    pass
+
+    def _flush_handlers(self):
+        """刷新所有handler"""
+        for handler in self.logger.handlers:
+            try:
+                handler.flush()
+            except Exception as e:
+                try:
+                    self.logger.error(f"刷新handler失败: {e}",
+                                      extra={'extra_data': {'handler_flush_error': str(e)}})
+                except:
+                    pass
+
+    def shutdown(self):
+        """关闭日志记录器，确保所有日志被刷新"""
+        if self.enable_async:
+            self._stop_event.set()
+            # 等待队列清空
+            while not self._log_queue.empty():
+                time.sleep(0.1)
+            if self._async_thread and self._async_thread.is_alive():
+                self._async_thread.join(timeout=0.5)
+
+        # 确保所有handler被刷新
+        if self._flush_thread:
+            self._flush_handlers()
+            if self._flush_thread.is_alive():
+                self._flush_thread.join(timeout=0.5)
+
+        # 关闭所有handler
+        for handler in self.logger.handlers:
+            try:
+                handler.close()
+            except:
+                pass
 
 def main():
     # 创建日志记录器
@@ -535,11 +631,11 @@ def main():
         name='my_app',
         logging_mode='both',
         log_level='DEBUG',
-        log_file='/Users/xigua/Downloads/my_app.log',
+        log_file='my_app.log',
         log_format='json',
         max_log_size=50,
         backup_count=5,
-        enable_async=True,  # 是否启用异步日志
+        enable_async=False,  # 是否启用异步日志
         sample_rate=1,  # 采样50%的DEBUG/INFO日志
         sensitive_fields=[],  #  敏感字段列表
         enable_metrics=False,  # 是否启用性能指标
@@ -553,4 +649,3 @@ def main():
 
 if __name__ == '__main__':
     pass
-    main()