mdbq 3.9.3__py3-none-any.whl → 3.9.5__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.9.3'
+VERSION = '3.9.5'

mdbq/log/mlogger.py

@@ -0,0 +1,531 @@
+import logging
+import logging.handlers
+import datetime
+import json
+import os
+import time
+import threading
+import queue
+from typing import Optional, Dict, Any, List, Callable, Union
+
+try:
+    import psutil
+
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+
+
+class MyLogger:
+    """
+    增强版结构化日志记录器，支持多种高级功能
+
+    功能增强：
+    - 异步日志记录（减少I/O阻塞）
+    - 上下文管理器支持
+    - 自定义日志过滤器
+    - 更丰富的系统指标采集
+    - 日志采样控制
+    - 动态日志级别调整
+    - 请求跟踪ID
+    - 多线程安全
+    - 日志缓冲和批量写入
+    - 自定义异常处理
+
+    使用示例：
+    logger = MyLogger(
+        name='app_logger',
+        logging_mode='both',
+        log_level='INFO',
+        log_file='app.log',
+        max_log_size=50,
+        backup_count=5,
+        enable_async=True
+    )
+
+    with logger.context(request_id='12345'):
+        logger.info("处理请求", extra={'user': 'admin'})
+    """
+
+    def __init__(
+            self,
+            name: str = 'mlogger',
+            logging_mode: str = 'console',  # 'both', 'console', 'file', 'none'
+            log_level: str = 'INFO',  # 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'
+            log_file: str = 'm_app.log',
+            log_format: str = 'json',  # 默认json格式，可选'simple'
+            max_log_size: int = 50,  # MB
+            backup_count: int = 5,
+            sensitive_fields: Optional[List[str]] = None,
+            enable_async: bool = False,
+            buffer_size: int = 1000,
+            sample_rate: float = 1.0,
+            filters: Optional[List[Callable]] = None,
+            enable_metrics: bool = False,
+            metrics_interval: int = 300,
+            message_limited: int = 1000
+    ):
+        """
+        初始化日志记录器
+
+        :param name: 日志记录器名称
+        :param logging_mode: 日志输出模式
+        :param log_level: 日志级别
+        :param log_file: 日志文件路径
+        :param max_log_size: 单个日志文件最大大小(MB)
+        :param backup_count: 保留的日志文件数量
+        :param sensitive_fields: 敏感字段列表(会被过滤)
+        :param enable_async: 是否启用异步日志
+        :param buffer_size: 日志缓冲大小(仅异步模式有效)
+        :param sample_rate: 控制日志的采样率(0.0-1.0)，使用消息内容的哈希值来决定是否记录，减少日志量，防止日志过于冗长
+        :param filters: 自定义日志过滤器列表
+        :param enable_metrics: 是否启用系统指标采集
+        :param metrics_interval: 指标采集间隔(秒)
+        :param message_limited: 简化日志内容，避免过长
+        """
+        self.name = name
+        self.logging_mode = logging_mode.lower()
+        self.log_level = log_level.upper()
+        self.log_file = log_file
+        self.log_format = log_format
+        self.max_log_size = max_log_size
+        self.backup_count = backup_count
+        self.sensitive_fields = sensitive_fields or []
+        self.enable_async = enable_async
+        self.buffer_size = buffer_size
+        self.sample_rate = max(0.0, min(1.0, sample_rate))
+        self.filters = filters or []
+        self.enable_metrics = enable_metrics and HAS_PSUTIL
+        self.metrics_interval = metrics_interval
+        self.message_limited = message_limited
+
+        # 上下文相关
+        self._context = threading.local()
+        self._context.data = {}
+
+        # 系统指标相关
+        self._last_metrics_time = 0
+        self._metrics_cache = {}
+
+        # 异步日志相关
+        self._log_queue = queue.Queue(maxsize=buffer_size)
+        self._async_thread = None
+        self._stop_event = threading.Event()
+
+        # 创建日志记录器
+        self.logger = logging.getLogger(name)
+        self._init_logging()
+
+        if self.enable_async:
+            self._start_async_logging()
+
+    def __enter__(self):
+        """上下文管理器入口"""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """上下文管理器退出"""
+        self.shutdown()
+        if exc_type is not None:
+            self.error(f"上下文内异常: {exc_val}",
+                       extra={'类型': str(exc_type)})
+        return False
+
+    def context(self, **kwargs):
+        """返回一个上下文管理器，可以设置临时上下文变量"""
+        return self._ContextManager(self, kwargs)
+
+    class _ContextManager:
+        def __init__(self, logger, context_vars):
+            self.logger = logger
+            self.context_vars = context_vars
+            self.old_context = {}
+
+        def __enter__(self):
+            # 保存旧上下文并设置新上下文
+            self.old_context = getattr(self.logger._context, 'data', {}).copy()
+            self.logger._context.data.update(self.context_vars)
+            return self.logger
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            # 恢复旧上下文
+            self.logger._context.data = self.old_context
+            if exc_type is not None:
+                self.logger.error(f"上下文内异常2: {exc_val}",
+                                  extra={'类型': str(exc_type)})
+            return False
+
+    def _init_logging(self):
+        """初始化日志配置"""
+        valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+        if self.log_level not in valid_levels:
+            self.log_level = 'INFO'
+
+        self.logger.setLevel(self.log_level)
+
+        # 防止重复添加handler
+        if self.logger.handlers:
+            for handler in self.logger.handlers[:]:
+                self.logger.removeHandler(handler)
+
+        # 定义日志格式
+        if self.log_format.lower() == 'simple':
+            # 简单文本格式
+            class SimpleFormatter(logging.Formatter):
+                def format(self, record):
+                    # 基础日志信息
+                    msg = super().format(record)
+
+                    # 添加上下文信息
+                    if hasattr(record, 'extra_data') and record.extra_data:
+                        context_data = record.extra_data.get('context_data', {})
+                        if context_data:
+                            msg += f" | Context: {context_data}"
+
+                        # 添加性能指标
+                        metrics = record.extra_data.get('性能指标', {})
+                        if metrics:
+                            msg += f" | Metrics: {metrics}"
+
+                        # 添加其他额外数据
+                        extra = {k: v for k, v in record.extra_data.items()
+                                 if k not in ('context_data', '性能指标')}
+                        if extra:
+                            msg += f" | Extra: {extra}"
+
+                    return msg
+
+            formatter = SimpleFormatter('%(asctime)s - %(levelname)s - %(message)s')
+            formatter.datefmt = '%Y-%m-%d %H:%M:%S'
+        else:
+            # 结构化JSON格式
+            class StructuredFormatter(logging.Formatter):
+                def format(self, record):
+                    log_data = {
+                        'timestamp': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                        'level': record.levelname,
+                        'message': record.getMessage(),
+                        'module': record.module,
+                        'function': record.funcName,
+                        'line': record.lineno,
+                        'thread': record.threadName,
+                        'process': record.processName,
+                    }
+
+                    # 添加额外字段
+                    if hasattr(record, 'extra_data'):
+                        log_data.update(record.extra_data)
+
+                    # 添加上下文信息
+                    if hasattr(record, 'context_data'):
+                        log_data.update(record.context_data)
+
+                    # 添加异常信息
+                    if record.exc_info:
+                        log_data['异常'] = self.formatException(record.exc_info)
+
+                    # 过滤敏感信息
+                    if hasattr(record, '过滤'):
+                        for field in record.sensitive_fields:
+                            if field in log_data:
+                                log_data[field] = '***'
+                            if isinstance(log_data.get('message'), str):
+                                log_data['message'] = log_data['message'].replace(field, '***')
+
+                    return json.dumps(log_data, ensure_ascii=False)
+
+            formatter = StructuredFormatter()
+
+        # 根据模式添加相应的handler
+        if self.logging_mode in ('both', 'console'):
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(formatter)
+            self.logger.addHandler(console_handler)
+
+        if self.logging_mode in ('both', 'file'):
+            # 确保日志目录存在
+            log_dir = os.path.dirname(self.log_file)
+            if log_dir and not os.path.exists(log_dir):
+                os.makedirs(log_dir)
+
+            file_handler = logging.handlers.RotatingFileHandler(
+                filename=self.log_file,
+                maxBytes=self.max_log_size * 1024 * 1024,
+                backupCount=self.backup_count,
+                encoding='utf-8'
+            )
+            file_handler.setFormatter(formatter)
+            self.logger.addHandler(file_handler)
+
+    def _get_system_metrics(self) -> Dict[str, Any]:
+        """获取系统资源使用指标"""
+        if not self.enable_metrics:
+            return {}
+
+        try:
+            return {
+                '内存': {
+                    '使用率': psutil.virtual_memory().percent,
+                    '已使用': psutil.virtual_memory().used,
+                    '可用': psutil.virtual_memory().available,
+                },
+                'CPU': {
+                    '使用率': psutil.cpu_percent(),
+                    '核心数': psutil.cpu_count(),
+                },
+                '磁盘': {
+                    '使用率': psutil.disk_usage('/').percent,
+                    '已使用': psutil.disk_usage('/').used,
+                    '剩余': psutil.disk_usage('/').free,
+                },
+                '网络': {
+                    '发送字节数': psutil.net_io_counters().bytes_sent,
+                    '接收字节数': psutil.net_io_counters().bytes_recv,
+                },
+                '进程': {
+                    'PID': os.getpid(),
+                    '线程数': threading.active_count(),
+                }
+            }
+        except Exception as e:
+            self.logger.warning(f"无法采集系统性能指标: {e}",
+                                extra={'extra_data': {'metrics_error': str(e)}})
+            return {}
+
+    def _apply_filters(self, level: str, message: str, extra: Dict) -> bool:
+        """应用自定义过滤器"""
+        for filter_func in self.filters:
+            try:
+                if not filter_func(level, message, extra):
+                    return False  # 如果过滤器返回 False，则丢弃该日志
+            except Exception as e:
+                self.logger.warning(f"过滤失败: {e}",
+                                    extra={'extra_data': {'filter_error': str(e)}})
+        return True  # 所有过滤器都返回 True，则记录该日志
+
+    def _async_log_worker(self):
+        """异步日志工作线程"""
+        while not self._stop_event.is_set() or not self._log_queue.empty():
+            try:
+                log_args = self._log_queue.get(timeout=0.1)
+                if log_args:
+                    level, message, extra = log_args
+                    self._sync_log(level, message, extra)
+            except queue.Empty:
+                continue
+            except Exception as e:
+                # 防止日志线程崩溃
+                try:
+                    self.logger.error(f"日志线程崩溃: {e}",
+                                      extra={'extra_data': {'async_error': str(e)}})
+                except:
+                    pass
+
+    def _start_async_logging(self):
+        """启动异步日志线程"""
+        self._stop_event.clear()
+        self._async_thread = threading.Thread(
+            target=self._async_log_worker,
+            name=f"{self.name}_async_logger",
+            daemon=True
+        )
+        self._async_thread.start()
+
+    def _sync_log(self, level: str, message: str, extra: Optional[Dict] = None):
+        """同步日志记录"""
+        if not hasattr(self.logger, level.lower()):
+            return
+
+        # 简化日志内容，避免过长
+        if len(message) > self.message_limited:
+            message = message[:self.message_limited] + '...'
+
+        # 定期收集系统指标
+        if self.enable_metrics:
+            now = time.time()
+            if now - self._last_metrics_time > self.metrics_interval:
+                self._metrics_cache = self._get_system_metrics()
+                self._last_metrics_time = now
+
+        # 准备日志额外数据
+        log_extra = {}
+        if self.enable_metrics:
+            log_extra['性能指标'] = self._metrics_cache
+        if extra:
+            log_extra.update(extra)
+
+        # 添加上下文信息
+        if hasattr(self._context, 'data') and self._context.data:
+            log_extra['context_data'] = self._context.data.copy()
+
+        # 添加敏感字段过滤
+        log_extra['过滤'] = self.sensitive_fields
+
+        # 应用日志采样
+        if self.sample_rate < 1.0 and level.lower() in ('debug', 'info'):
+            if hash(message) % 100 >= self.sample_rate * 100:
+                return
+
+        # 应用过滤器
+        if not self._apply_filters(level, message, log_extra):
+            return
+
+        # 记录日志
+        getattr(self.logger, level.lower())(message, extra={'extra_data': log_extra})
+
+    def log(self, level: str, message: str, extra: Optional[Dict] = None):
+        """
+        记录日志
+
+        :param level: 日志级别 ('debug', 'info', 'warning', 'error', 'critical')
+        :param message: 日志消息
+        :param extra: 额外数据字典
+        """
+        if not hasattr(self.logger, level.lower()):
+            return
+
+        if self.enable_async:
+            try:
+                self._log_queue.put((level, message, extra), timeout=0.1)
+            except queue.Full:
+                # 队列满时降级为同步日志
+                self._sync_log(level, f"[ASYNC QUEUE FULL] {message}", extra)
+        else:
+            self._sync_log(level, message, extra)
+
+    def set_level(self, level: str):
+        """动态设置日志级别"""
+        valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+        level = level.upper()
+        if level in valid_levels:
+            self.log_level = level
+            self.logger.setLevel(level)
+            for handler in self.logger.handlers:
+                handler.setLevel(level)
+
+    def add_filter(self, filter_func: Callable):
+        """添加日志过滤器"""
+        if callable(filter_func):
+            self.filters.append(filter_func)
+
+    def set_context(self, **kwargs):
+        """设置上下文变量"""
+        if not hasattr(self._context, 'data'):
+            self._context.data = {}
+        self._context.data.update(kwargs)
+
+    def get_context(self, key: str, default=None):
+        """获取上下文变量"""
+        if hasattr(self._context, 'data'):
+            return self._context.data.get(key, default)
+        return default
+
+    def clear_context(self):
+        """清除所有上下文变量"""
+        if hasattr(self._context, 'data'):
+            self._context.data.clear()
+
+    def shutdown(self):
+        """关闭日志记录器，确保所有日志被刷新"""
+        if self.enable_async:
+            self._stop_event.set()
+            if self._async_thread and self._async_thread.is_alive():
+                self._async_thread.join(timeout=5)
+
+        # 确保所有handler被刷新
+        for handler in self.logger.handlers:
+            handler.flush()
+            handler.close()
+
+    def debug(self, message: str, extra: Optional[Dict] = None):
+        """记录调试信息"""
+        self.log('debug', message, extra)
+
+    def info(self, message: str, extra: Optional[Dict] = None):
+        """记录一般信息"""
+        self.log('info', message, extra)
+
+    def warning(self, message: str, extra: Optional[Dict] = None):
+        """记录警告信息"""
+        self.log('warning', message, extra)
+
+    def error(self, message: str, extra: Optional[Dict] = None):
+        """记录错误信息"""
+        self.log('error', message, extra)
+
+    def critical(self, message: str, extra: Optional[Dict] = None):
+        """记录严重错误信息"""
+        self.log('critical', message, extra)
+
+    def exception(self, message: str, exc_info: Exception, extra: Optional[Dict] = None):
+        """记录异常信息"""
+        if not extra:
+            extra = {}
+        extra['异常'] = str(exc_info)
+        extra['类型'] = exc_info.__class__.__name__
+        self.log('error', message, extra)
+
+    def timeit(self, message: str = "Execution time"):
+        """返回一个计时器上下文管理器"""
+        return self._Timer(self, message)
+
+    class _Timer:
+        def __init__(self, logger, message):
+            self.logger = logger
+            self.message = message
+            self.start_time = None
+
+        def __enter__(self):
+            self.start_time = time.time()
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            elapsed = time.time() - self.start_time
+            self.logger.info(f"{self.message}: {elapsed:.3f}s",
+                             extra={'elapsed_seconds': elapsed})
+            return False
+
+
+def main():
+    # 创建日志记录器
+    logger = MyLogger(
+        name='my_app',
+        logging_mode='both',
+        log_level='DEBUG',
+        log_file='/Users/xigua/Downloads/my_app.log',
+        log_format='simple',
+        enable_async=True,
+        sample_rate=1,  # 采样50%的DEBUG/INFO日志
+        sensitive_fields=['password', 'token'],
+        enable_metrics=True
+    )
+
+    # 添加自定义过滤器
+    def keyword_filter(level, message, extra):
+        blocked_keywords = ['secret', 'password']
+        return not any(keyword in message.lower() for keyword in blocked_keywords)
+
+    logger.add_filter(keyword_filter)
+
+    # 使用上下文管理器
+    with logger.context(request_id='12345', user='admin'):
+        logger.info("开始处理请求")
+
+        # 使用计时器
+        with logger.timeit("数据库查询"):
+            time.sleep(0.1)  # 模拟耗时操作
+
+        logger.debug("调试信息", extra={'data': {'key': 'value'}})
+
+        try:
+            1 / 0
+        except Exception as e:
+            logger.exception("发生错误", e)
+
+    # 确保所有日志被刷新
+    logger.shutdown()
+
+
+if __name__ == '__main__':
+    pass
+    main()

mdbq/mysql/mysql.py

@@ -12,11 +12,14 @@ import os
 import logging
 import logging.handlers
 from mdbq.other import otk
-from typing import Union, List, Dict, Optional, Any, Tuple
+from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
 import json
 import psutil
 from collections import OrderedDict
+import threading
+import concurrent.futures
+from collections import defaultdict
 
 
 warnings.filterwarnings('ignore')
@@ -2383,10 +2386,625 @@ class MySQLUploader:
 
         return decorator
 
+
+class MySQLDeduplicator:
+    """
+    MySQL数据去重
+
+    功能：
+    1. 自动检测并删除MySQL数据库中的重复数据
+    2. 支持全库扫描或指定表处理
+    3. 支持多线程/多进程安全处理
+    4. 完善的错误处理和日志记录
+
+    使用示例：
+    deduplicator = MySQLDeduplicator(
+        username='root',
+        password='password',
+        host='localhost',
+        port=3306
+    )
+
+    # 全库去重
+    deduplicator.deduplicate_all()
+
+    # 指定数据库去重(多线程)
+    deduplicator.deduplicate_database('my_db', parallel=True)
+
+    # 指定表去重(使用特定列)
+    deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
+
+    # 关闭连接
+    deduplicator.close()
+    """
+
+    def __init__(
+            self,
+            username: str,
+            password: str,
+            host: str = 'localhost',
+            port: int = 3306,
+            charset: str = 'utf8mb4',
+            max_workers: int = 1,
+            batch_size: int = 1000,
+            skip_system_dbs: bool = True,
+            logging_mode: str = 'console',
+            log_level: str = 'INFO',
+            log_file: str = 'mysql_deduplicate.log',
+            max_retries: int = 3,
+            retry_interval: int = 5,
+            pool_size: int = 5
+    ):
+        """
+        初始化去重处理器
+
+        :param username: 数据库用户名
+        :param password: 数据库密码
+        :param host: 数据库主机，默认为localhost
+        :param port: 数据库端口，默认为3306
+        :param charset: 字符集，默认为utf8mb4
+        :param max_workers: 最大工作线程数，默认为1(单线程)
+        :param batch_size: 批量处理大小，默认为1000
+        :param skip_system_dbs: 是否跳过系统数据库，默认为True
+        :param logging_mode: 日志模式('console', 'file', 'both', 'none')
+        :param log_level: 日志级别('DEBUG', 'INFO', 'WARNING', 'ERROR')
+        :param log_file: 日志文件路径
+        :param max_retries: 最大重试次数
+        :param retry_interval: 重试间隔(秒)
+        :param pool_size: 连接池大小
+        """
+        # 初始化连接池
+        self.pool = PooledDB(
+            creator=pymysql,
+            host=host,
+            port=port,
+            user=username,
+            password=password,
+            charset=charset,
+            maxconnections=pool_size,
+            cursorclass=pymysql.cursors.DictCursor
+        )
+
+        # 配置参数
+        self.max_workers = max(1, min(max_workers, 20))  # 限制最大线程数
+        self.batch_size = batch_size
+        self.skip_system_dbs = skip_system_dbs
+        self.max_retries = max_retries
+        self.retry_interval = retry_interval
+
+        # 线程安全控制
+        self._lock = threading.Lock()
+        self._processing_tables = set()  # 正在处理的表集合
+
+        # 初始化日志
+        self._init_logging(logging_mode, log_level, log_file)
+
+        # 系统数据库列表
+        self.SYSTEM_DATABASES = {
+            'information_schema', 'mysql',
+            'performance_schema', 'sys'
+        }
+
+    def _init_logging(
+            self,
+            logging_mode: str,
+            log_level: str,
+            log_file: str
+    ):
+        """初始化日志配置"""
+        self.logger = logging.getLogger('mysql_deduplicator')
+        self.logger.setLevel(log_level.upper())
+
+        # 防止重复添加handler
+        if self.logger.handlers:
+            for handler in self.logger.handlers[:]:
+                self.logger.removeHandler(handler)
+
+        formatter = logging.Formatter(
+            '%(asctime)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+        mode = logging_mode.lower()
+        if mode in ('both', 'console'):
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(formatter)
+            self.logger.addHandler(console_handler)
+
+        if mode in ('both', 'file'):
+            file_handler = logging.FileHandler(
+                filename=log_file,
+                encoding='utf-8'
+            )
+            file_handler.setFormatter(formatter)
+            self.logger.addHandler(file_handler)
+
+    def _log(self, level: str, message: str, extra: Optional[Dict] = None):
+        """统一的日志记录方法"""
+        if not hasattr(self.logger, level.lower()):
+            return
+
+        # 简化日志内容，避免过长
+        if len(message) > 500:
+            message = message[:500] + '...'
+
+        log_method = getattr(self.logger, level.lower())
+        log_method(message, extra=extra)
+
+    def _get_connection(self):
+        """从连接池获取连接"""
+        try:
+            conn = self.pool.connection()
+            self._log('debug', "成功获取数据库连接")
+            return conn
+        except Exception as e:
+            self._log('error', f"获取数据库连接失败: {str(e)}")
+            raise ConnectionError(f"连接数据库失败: {str(e)}")
+
+    @staticmethod
+    def _retry_on_failure(func):
+        """重试装饰器"""
+
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            last_exception = None
+            for attempt in range(self.max_retries + 1):
+                try:
+                    return func(self, *args, **kwargs)
+                except (pymysql.OperationalError, pymysql.InterfaceError) as e:
+                    last_exception = e
+                    if attempt < self.max_retries:
+                        wait_time = self.retry_interval * (attempt + 1)
+                        self._log('warning',
+                                  f"数据库操作失败，准备重试 (尝试 {attempt + 1}/{self.max_retries})",
+                                  {'error': str(e), 'wait_time': wait_time})
+                        time.sleep(wait_time)
+                        continue
+                except Exception as e:
+                    last_exception = e
+                    self._log('error',
+                              f"操作失败: {str(e)}",
+                              {'error_type': type(e).__name__})
+                    break
+
+            if last_exception:
+                raise last_exception
+            raise Exception("未知错误")
+
+        return wrapper
+
+    @_retry_on_failure
+    def _get_databases(self) -> List[str]:
+        """获取所有非系统数据库列表"""
+        sql = "SHOW DATABASES"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql)
+                all_dbs = [row['Database'] for row in cursor.fetchall()]
+
+                if self.skip_system_dbs:
+                    return [db for db in all_dbs if db.lower() not in self.SYSTEM_DATABASES]
+                return all_dbs
+
+    @_retry_on_failure
+    def _get_tables(self, database: str) -> List[str]:
+        """获取指定数据库的所有表"""
+        sql = "SHOW TABLES"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(f"USE `{database}`")
+                cursor.execute(sql)
+                return [row[f'Tables_in_{database}'] for row in cursor.fetchall()]
+
+    @_retry_on_failure
+    def _get_table_columns(self, database: str, table: str) -> List[str]:
+        """获取表的列名(排除id列)"""
+        sql = """
+        SELECT COLUMN_NAME 
+        FROM INFORMATION_SCHEMA.COLUMNS 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        ORDER BY ORDINAL_POSITION
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (database, table))
+                return [row['COLUMN_NAME'] for row in cursor.fetchall()
+                        if row['COLUMN_NAME'].lower() != 'id']
+
+    def _acquire_table_lock(self, database: str, table: str) -> bool:
+        """获取表处理锁，防止并发处理同一张表"""
+        key = f"{database}.{table}"
+
+        with self._lock:
+            if key in self._processing_tables:
+                self._log('debug', f"表 {key} 正在被其他线程处理，跳过")
+                return False
+            self._processing_tables.add(key)
+            return True
+
+    def _release_table_lock(self, database: str, table: str):
+        """释放表处理锁"""
+        key = f"{database}.{table}"
+
+        with self._lock:
+            if key in self._processing_tables:
+                self._processing_tables.remove(key)
+
+    def _deduplicate_table(
+            self,
+            database: str,
+            table: str,
+            columns: Optional[List[str]] = None,
+            dry_run: bool = False
+    ) -> Tuple[int, int]:
+        """
+        执行单表去重
+
+        :param database: 数据库名
+        :param table: 表名
+        :param columns: 用于去重的列(为None时使用所有列)
+        :param dry_run: 是否模拟运行(只统计不实际删除)
+        :return: (重复行数, 删除行数)
+        """
+        if not self._acquire_table_lock(database, table):
+            return (0, 0)
+
+        try:
+            self._log('info', f"开始处理表: {database}.{table}")
+
+            # 获取实际列名
+            all_columns = self._get_table_columns(database, table)
+            if not all_columns:
+                self._log('warning', f"表 {database}.{table} 没有有效列(可能只有id列)，跳过")
+                return (0, 0)
+
+            # 使用指定列或所有列
+            use_columns = columns or all_columns
+            invalid_columns = set(use_columns) - set(all_columns)
+
+            if invalid_columns:
+                self._log('warning',
+                          f"表 {database}.{table} 中不存在以下列: {invalid_columns}，使用有效列",
+                          {'invalid_columns': invalid_columns}
+                          )
+                use_columns = [col for col in use_columns if col in all_columns]
+
+            if not use_columns:
+                self._log('error', f"表 {database}.{table} 没有有效的去重列")
+                return (0, 0)
+
+            # 构建去重SQL
+            column_list = ', '.join([f'`{col}`' for col in use_columns])
+            temp_table = f"temp_{table}_{int(time.time())}"
+
+            # 使用临时表方案处理去重，避免锁表问题
+            create_temp_sql = f"""
+            CREATE TABLE `{database}`.`{temp_table}` AS
+            SELECT MIN(`id`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
+            FROM `{database}`.`{table}`
+            GROUP BY {column_list}
+            HAVING COUNT(*) > 1
+            """
+
+            delete_dup_sql = f"""
+            DELETE FROM `{database}`.`{table}`
+            WHERE `id` NOT IN (
+                SELECT `min_id` FROM `{database}`.`{temp_table}`
+            ) AND ({' OR '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
+            """
+
+            drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
+
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    # 创建临时表统计重复数据
+                    cursor.execute(create_temp_sql)
+                    cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`")
+                    dup_count = cursor.fetchone()['cnt']
+
+                    if dup_count == 0:
+                        self._log('info', f"表 {database}.{table} 没有重复数据")
+                        cursor.execute(drop_temp_sql)
+                        conn.commit()
+                        return (0, 0)
+
+                    self._log('info',
+                              f"表 {database}.{table} 发现 {dup_count} 组重复数据",
+                              {'columns': use_columns}
+                              )
+
+                    if not dry_run:
+                        # 执行实际删除
+                        cursor.execute(delete_dup_sql)
+                        affected_rows = cursor.rowcount
+                        conn.commit()
+                        self._log('info',
+                                  f"表 {database}.{table} 已删除 {affected_rows} 行重复数据",
+                                  {'columns': use_columns}
+                                  )
+                    else:
+                        affected_rows = 0
+                        self._log('info',
+                                  f"[模拟运行] 表 {database}.{table} 将删除 {dup_count} 组重复数据",
+                                  {'columns': use_columns}
+                                  )
+
+                    # 清理临时表
+                    cursor.execute(drop_temp_sql)
+                    conn.commit()
+
+                    return (dup_count, affected_rows)
+
+        except Exception as e:
+            self._log('error',
+                      f"处理表 {database}.{table} 时出错: {str(e)}",
+                      {'error_type': type(e).__name__}
+                      )
+            return (0, 0)
+        finally:
+            self._release_table_lock(database, table)
+
+    def deduplicate_table(
+            self,
+            database: str,
+            table: str,
+            columns: Optional[List[str]] = None,
+            dry_run: bool = False
+    ) -> Tuple[int, int]:
+        """
+        对指定表进行去重
+
+        :param database: 数据库名
+        :param table: 表名
+        :param columns: 用于去重的列(为None时使用所有列)
+        :param dry_run: 是否模拟运行(只统计不实际删除)
+        :return: (重复行数, 删除行数)
+        """
+        try:
+            # 检查表是否存在
+            if not self._check_table_exists(database, table):
+                self._log('warning', f"表 {database}.{table} 不存在，跳过")
+                return (0, 0)
+
+            return self._deduplicate_table(database, table, columns, dry_run)
+        except Exception as e:
+            self._log('error',
+                      f"处理表 {database}.{table} 时发生全局错误: {str(e)}",
+                      {'error_type': type(e).__name__}
+                      )
+            return (0, 0)
+
+    def deduplicate_database(
+            self,
+            database: str,
+            tables: Optional[List[str]] = None,
+            columns_map: Optional[Dict[str, List[str]]] = None,
+            dry_run: bool = False,
+            parallel: bool = False
+    ) -> Dict[str, Tuple[int, int]]:
+        """
+        对指定数据库的所有表进行去重
+
+        :param database: 数据库名
+        :param tables: 要处理的表列表(为None时处理所有表)
+        :param columns_map: 各表使用的去重列 {表名: [列名]}
+        :param dry_run: 是否模拟运行
+        :param parallel: 是否并行处理
+        :return: 字典 {表名: (重复行数, 删除行数)}
+        """
+        results = {}
+
+        try:
+            # 检查数据库是否存在
+            if not self._check_database_exists(database):
+                self._log('warning', f"数据库 {database} 不存在，跳过")
+                return results
+
+            # 获取要处理的表
+            target_tables = tables or self._get_tables(database)
+            if not target_tables:
+                self._log('info', f"数据库 {database} 中没有表，跳过")
+                return results
+
+            self._log('info',
+                      f"开始处理数据库 {database} 中的 {len(target_tables)} 张表",
+                      {'tables': target_tables}
+                      )
+
+            if parallel and self.max_workers > 1:
+                # 并行处理
+                with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=self.max_workers
+                ) as executor:
+                    futures = {}
+                    for table in target_tables:
+                        columns = columns_map.get(table) if columns_map else None
+                        futures[executor.submit(
+                            self.deduplicate_table,
+                            database, table, columns, dry_run
+                        )] = table
+
+                    for future in concurrent.futures.as_completed(futures):
+                        table = futures[future]
+                        try:
+                            dup_count, affected_rows = future.result()
+                            results[table] = (dup_count, affected_rows)
+                        except Exception as e:
+                            self._log('error',
+                                      f"处理表 {database}.{table} 时出错: {str(e)}",
+                                      {'error_type': type(e).__name__}
+                                      )
+                            results[table] = (0, 0)
+            else:
+                # 串行处理
+                for table in target_tables:
+                    columns = columns_map.get(table) if columns_map else None
+                    dup_count, affected_rows = self.deduplicate_table(
+                        database, table, columns, dry_run
+                    )
+                    results[table] = (dup_count, affected_rows)
+
+            # 统计结果
+            total_dup = sum(r[0] for r in results.values())
+            total_del = sum(r[1] for r in results.values())
+
+            self._log('info',
+                      f"数据库 {database} 处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
+                      {'results': results}
+                      )
+
+            return results
+
+        except Exception as e:
+            self._log('error',
+                      f"处理数据库 {database} 时发生全局错误: {str(e)}",
+                      {'error_type': type(e).__name__}
+                      )
+            return results
+
+    def deduplicate_all(
+            self,
+            databases: Optional[List[str]] = None,
+            tables_map: Optional[Dict[str, List[str]]] = None,
+            columns_map: Optional[Dict[str, Dict[str, List[str]]]] = None,
+            dry_run: bool = False,
+            parallel: bool = False
+    ) -> Dict[str, Dict[str, Tuple[int, int]]]:
+        """
+        对所有数据库进行去重
+
+        :param databases: 要处理的数据库列表(为None时处理所有非系统数据库)
+        :param tables_map: 各数据库要处理的表 {数据库名: [表名]}
+        :param columns_map: 各表使用的去重列 {数据库名: {表名: [列名]}}
+        :param dry_run: 是否模拟运行
+        :param parallel: 是否并行处理
+        :return: 嵌套字典 {数据库名: {表名: (重复行数, 删除行数)}}
+        """
+        all_results = defaultdict(dict)
+
+        try:
+            # 获取要处理的数据库
+            target_dbs = databases or self._get_databases()
+            if not target_dbs:
+                self._log('warning', "没有可处理的数据库")
+                return all_results
+
+            self._log('info',
+                      f"开始处理 {len(target_dbs)} 个数据库",
+                      {'databases': target_dbs}
+                      )
+
+            if parallel and self.max_workers > 1:
+                # 并行处理数据库
+                with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=self.max_workers
+                ) as executor:
+                    futures = {}
+                    for db in target_dbs:
+                        tables = tables_map.get(db) if tables_map else None
+                        db_columns_map = columns_map.get(db) if columns_map else None
+                        futures[executor.submit(
+                            self.deduplicate_database,
+                            db, tables, db_columns_map, dry_run, False
+                        )] = db
+
+                    for future in concurrent.futures.as_completed(futures):
+                        db = futures[future]
+                        try:
+                            db_results = future.result()
+                            all_results[db] = db_results
+                        except Exception as e:
+                            self._log('error',
+                                      f"处理数据库 {db} 时出错: {str(e)}",
+                                      {'error_type': type(e).__name__}
+                                      )
+                            all_results[db] = {}
+            else:
+                # 串行处理数据库
+                for db in target_dbs:
+                    tables = tables_map.get(db) if tables_map else None
+                    db_columns_map = columns_map.get(db) if columns_map else None
+                    db_results = self.deduplicate_database(
+                        db, tables, db_columns_map, dry_run, parallel
+                    )
+                    all_results[db] = db_results
+
+            # 统计总体结果
+            total_dup = sum(
+                r[0] for db in all_results.values()
+                for r in db.values()
+            )
+            total_del = sum(
+                r[1] for db in all_results.values()
+                for r in db.values()
+            )
+
+            self._log('info',
+                      f"所有数据库处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
+                      {'total_results': all_results}
+                      )
+
+            return all_results
+
+        except Exception as e:
+            self._log('error',
+                      f"全局处理时发生错误: {str(e)}",
+                      {'error_type': type(e).__name__}
+                      )
+            return all_results
+
+    @_retry_on_failure
+    def _check_database_exists(self, database: str) -> bool:
+        """检查数据库是否存在"""
+        sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (database,))
+                return bool(cursor.fetchone())
+
+    @_retry_on_failure
+    def _check_table_exists(self, database: str, table: str) -> bool:
+        """检查表是否存在"""
+        sql = """
+        SELECT TABLE_NAME 
+        FROM INFORMATION_SCHEMA.TABLES 
+        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+        """
+
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql, (database, table))
+                return bool(cursor.fetchone())
+
+    def close(self):
+        """关闭连接池"""
+        try:
+            if hasattr(self, 'pool') and self.pool:
+                self.pool.close()
+                self._log('info', "数据库连接池已关闭")
+        except Exception as e:
+            self._log('error',
+                      f"关闭连接池时出错: {str(e)}",
+                      {'error_type': type(e).__name__}
+                      )
+        finally:
+            self.pool = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
 def main():
     uploader = MySQLUploader(
         username='root',
-        password='1',
+        password='188988yang188',
         host='localhost',
         port=3306,
         logging_mode='console',
@@ -2417,7 +3035,7 @@ def main():
         data=data,
         set_typ=set_typ,  # 定义列和数据类型
         primary_keys=[],  # 创建唯一主键
-        check_duplicate=True,  # 检查重复数据
+        check_duplicate=False,  # 检查重复数据
         duplicate_columns=[],  # 指定排重的组合键
         allow_null=False,  # 允许插入空值
         partition_by='year',  # 按月分表
@@ -2429,5 +3047,27 @@ def main():
     uploader.close()
 
 
+def main2():
+    deduplicator = MySQLDeduplicator(
+        username='root',
+        password='1',
+        host='localhost',
+        port=3306
+    )
+
+    # # 全库去重(单线程)
+    # deduplicator.deduplicate_all()
+
+    # # 指定数据库去重(多线程)
+    # deduplicator.deduplicate_database('my_db', parallel=True)
+
+    # 指定表去重(使用特定列)
+    deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
+
+    # 关闭连接
+    deduplicator.close()
+
 if __name__ == '__main__':
     pass
+
+    main2()

{mdbq-3.9.3.dist-info → mdbq-3.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.3
+Version: 3.9.5
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.9.3.dist-info → mdbq-3.9.5.dist-info}/RECORD

@@ -1,14 +1,15 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=Pw2FixsnE8Hf360X55h_tF8Xeez7UgHd2pSgUkJY-v4,17
+mdbq/__version__.py,sha256=6a9QWBZBLdUXab5LGTsteurYczwlY3DSfMu9kjZZUlA,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
+mdbq/log/mlogger.py,sha256=eM_0CYSWZsiwl6jHkm9F7S7pb_V54LdImrC3tOtxojg,19686
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=YX-tgugceODrJHcXgbosWFVThjXv3I2gCvTt_siKBOI,108606
+mdbq/mysql/mysql.py,sha256=WFkv2flWuYRJqTzCOvxyiPeJgyJvVQzSG24jH_x5oEg,132390
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
@@ -22,7 +23,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
-mdbq-3.9.3.dist-info/METADATA,sha256=Vt2mII7wAfEhzQa9G8PreCPV_hkdM1DLTTDcUMyepPg,363
-mdbq-3.9.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.9.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.9.3.dist-info/RECORD,,
+mdbq-3.9.5.dist-info/METADATA,sha256=uv8f_2V0nVGSA9GFa39R9Pdg1ZX2O8ACLQnZkjR9T7I,363
+mdbq-3.9.5.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.5.dist-info/RECORD,,