mdbq 3.11.10__py3-none-any.whl → 3.12.0__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.11.10'
+VERSION = '3.12.0'

mdbq/mysql/deduplicator.py

@@ -14,6 +14,7 @@ from collections import defaultdict
 import sys
 from datetime import datetime
 import uuid
+from contextlib import contextmanager
 
 
 warnings.filterwarnings('ignore')
@@ -34,32 +35,6 @@ logger = mylogger.MyLogger(
 class MySQLDeduplicator:
     """
     MySQL数据去重
-
-    功能：
-    1. 自动检测并删除MySQL数据库中的重复数据
-    2. 支持全库扫描或指定表理
-    3. 支持多线程/多进程安全处理
-    4. 完善的错误处理和日志记录
-
-    使用示例：
-    deduplicator = MySQLDeduplicator(
-        username='root',
-        password='password',
-        host='localhost',
-        port=3306
-    )
-
-    # 全库去重
-    deduplicator.deduplicate_all()
-
-    # 指定数据库去重(多线程)
-    deduplicator.deduplicate_database('my_db', parallel=True)
-
-    # 指定表去重(使用特定列)
-    deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
-
-    # 关闭连接
-    deduplicator.close()
     """
 
     def __init__(
@@ -69,12 +44,12 @@ class MySQLDeduplicator:
             host: str = 'localhost',
             port: int = 3306,
             charset: str = 'utf8mb4',
-            max_workers: int = 1,
+            max_workers: int = 2,
             batch_size: int = 1000,
             skip_system_dbs: bool = True,
             max_retries: int = 3,
-            retry_interval: int = 5,
-            pool_size: int = 5,
+            retry_waiting_time: int = 5,
+            pool_size: int = 10,
             primary_key: str = 'id',
             date_range: Optional[List[str]] = None,
             recent_month: Optional[int] = None,
@@ -113,35 +88,40 @@ class MySQLDeduplicator:
             cursorclass=pymysql.cursors.DictCursor
         )
 
+        # 并发模式要将 pool_size 加大
+        MAX_POOL_SIZE = 200
+        MAX_WORKERS = 4
+        if max_workers > MAX_WORKERS:
+            logger.warning(f"max_workers({max_workers}) 超过最大建议值({MAX_WORKERS})，自动将 max_workers 调整为 {MAX_WORKERS}")
+            max_workers = MAX_WORKERS
+        expected_threads = max_workers * 10
+        if pool_size < expected_threads:
+            logger.warning(f"pool_size({pool_size}) < max_workers({max_workers}) * 10，自动将 pool_size 调整为 {expected_threads}")
+            pool_size = expected_threads
+        if pool_size > MAX_POOL_SIZE:
+            logger.warning(f"pool_size({pool_size}) 超过最大建议值({MAX_POOL_SIZE})，自动将 pool_size 调整为 {MAX_POOL_SIZE}")
+            pool_size = MAX_POOL_SIZE
+        self.max_workers = max_workers
+        self.pool_size = pool_size
+
         # 配置参数
-        self.max_workers = min(max(1, max_workers), pool_size)  # 限制最大线程数，不能超过连接池
         self.batch_size = batch_size
         self.skip_system_dbs = skip_system_dbs
         self.max_retries = max_retries
-        self.retry_interval = retry_interval
+        self.retry_waiting_time = retry_waiting_time
         self.primary_key = primary_key
 
         # 时间范围参数
-        self.date_range = date_range
-        self.recent_month = recent_month
         self.date_column = date_column
         self._dedup_start_date = None
         self._dedup_end_date = None
-        # 不管 exclude_columns 是否传入, 'id' 一定会被排除
-        default_exclude = {'id'}
-        # exclude_columns 不传则排除: ['id', '更新时间']
-        if not exclude_columns:
-            self.exclude_columns = list(default_exclude | {'更新时间'})
-        else:
-            self.exclude_columns = list(set(exclude_columns) | default_exclude)
-        # 解析时间范围并智能校正date_range
-        if self.date_range and len(self.date_range) == 2:
+        if date_range and len(date_range) == 2:
             try:
-                start, end = self.date_range
+                start, end = date_range
                 start_dt = datetime.strptime(start, "%Y-%m-%d")
                 end_dt = datetime.strptime(end, "%Y-%m-%d")
                 if start_dt > end_dt:
-                    logger.warning(
+                    logger.debug(
                         "date_range顺序不正确，自动交换开始和结束日期。",
                         {"start": start, "end": end}
                     )
@@ -151,30 +131,36 @@ class MySQLDeduplicator:
             except Exception as e:
                 logger.error(
                     "date_range参数格式错误，应为['YYYY-MM-DD', 'YYYY-MM-DD']，已忽略时间范围。",
-                    {"date_range": self.date_range, "error": str(e)}
+                    {"date_range": date_range, "error": str(e)}
                 )
                 self._dedup_start_date = None
                 self._dedup_end_date = None
-        elif self.recent_month:
+        elif recent_month:
             today = datetime.today()
-            month = today.month - self.recent_month
+            month = today.month - recent_month
             year = today.year
             while month <= 0:
                 month += 12
                 year -= 1
             self._dedup_start_date = f"{year}-{month:02d}-01"
             self._dedup_end_date = today.strftime("%Y-%m-%d")
+        
+        if self._dedup_start_date and self._dedup_end_date:
+            logger.info('去重日期范围', {'开始': self._dedup_start_date, '结束': self._dedup_end_date})
+
+        # 排除列处理，直接合并去重
+        self.exclude_columns = list(set((exclude_columns or []) + ['id', '更新时间']))
 
         # 线程安全控制
         self._lock = threading.Lock()
         self._processing_tables = set()  # 正在处理的表集合
 
         # 系统数据库列表
-        self.SYSTEM_DATABASES = {'information_schema', 'mysql', 'performance_schema', 'sys'}
+        self.SYSTEM_DATABASES = {'information_schema', 'mysql', 'performance_schema', 'sys', 'sakila'}
 
         # 排除数据库和表的逻辑
-        self.exclude_databases = set([db.lower() for db in exclude_databases]) if exclude_databases else set()
-        self.exclude_tables = {k.lower(): set([t.lower() for t in v]) for k, v in (exclude_tables or {}).items()}
+        self.exclude_databases = set(db.lower() for db in (exclude_databases or []))
+        self.exclude_tables = {k.lower(): set(t.lower() for t in v) for k, v in (exclude_tables or {}).items()}
 
         self.duplicate_keep_mode = duplicate_keep_mode if duplicate_keep_mode in ('keep_one', 'remove_all') else 'keep_one'
 
@@ -197,6 +183,14 @@ class MySQLDeduplicator:
             logger.error(f"获取数据库连接失败: {str(e)}", {'error_type': type(e).__name__})
             raise ConnectionError(f"连接数据库失败: {str(e)}")
 
+    @contextmanager
+    def _conn_ctx(self):
+        conn = self._get_connection()
+        try:
+            yield conn
+        finally:
+            conn.close()
+
     @staticmethod
     def _retry_on_failure(func: Any) -> Any:
         """
@@ -220,7 +214,7 @@ class MySQLDeduplicator:
                 except (pymysql.OperationalError, pymysql.InterfaceError) as e:
                     last_exception = e
                     if attempt < self.max_retries:
-                        wait_time = self.retry_interval * (attempt + 1)
+                        wait_time = self.retry_waiting_time * (attempt + 1)
                         logger.warning(
                             f"数据库操作失败，准备重试 (尝试 {attempt + 1}/{self.max_retries})",
                             {'error': str(e), 'wait_time': wait_time, 'func': func.__name__})
@@ -236,16 +230,15 @@ class MySQLDeduplicator:
             raise Exception("未知错误")
         return wrapper
 
-    @_retry_on_failure
     def _get_databases(self) -> List[str]:
         """
-        获取所有非系统数据库列表，排除exclude_databases。
+        获取所有非系统数据库列表，排除 exclude_databases。
         
         Returns:
             List[str]: 数据库名列表。
         """
         sql = "SHOW DATABASES"
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(sql)
                 all_dbs = [row['Database'] for row in cursor.fetchall()]
@@ -253,10 +246,9 @@ class MySQLDeduplicator:
                 filtered = [db for db in all_dbs if db.lower() not in self.SYSTEM_DATABASES and db.lower() not in self.exclude_databases] if self.skip_system_dbs else [db for db in all_dbs if db.lower() not in self.exclude_databases]
                 return filtered
 
-    @_retry_on_failure
     def _get_tables(self, database: str) -> List[str]:
         """
-        获取指定数据库的所有表名。
+        获取指定数据库的所有表名（排除 temp_ 前缀的临时表）。
         
         Args:
             database (str): 数据库名。
@@ -264,15 +256,12 @@ class MySQLDeduplicator:
             List[str]: 表名列表。
         """
         sql = "SHOW TABLES"
-
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(f"USE `{database}`")
                 cursor.execute(sql)
-                # 严格过滤所有以'temp_'为前缀的表名（如temp_xxx、temp_xxx_dedup_...、temp_xxx_reorderid_...等）
                 return [row[f'Tables_in_{database}'] for row in cursor.fetchall() if not re.match(r'^temp_.*', row[f'Tables_in_{database}'])]
 
-    @_retry_on_failure
     def _get_table_columns(self, database: str, table: str) -> List[str]:
         """
         获取指定表的所有列名（排除主键列）。
@@ -289,56 +278,22 @@ class MySQLDeduplicator:
         WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
         ORDER BY ORDINAL_POSITION
         """
-
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(sql, (database, table))
                 return [row['COLUMN_NAME'] for row in cursor.fetchall()
                         if row['COLUMN_NAME'].lower() != self.primary_key.lower()]
 
-    def _acquire_table_lock(self, database: str, table: str) -> bool:
-        """
-        获取表处理锁，防止并发处理同一张表。
-        
-        Args:
-            database (str): 数据库名。
-            table (str): 表名。
-        Returns:
-            bool: 是否成功获取锁。
-        """
-        key = f"{database}.{table}"
-
-        with self._lock:
-            if key in self._processing_tables:
-                logger.debug(f"表 {key} 正在被其他线程处理，跳过")
-                return False
-            self._processing_tables.add(key)
-            return True
-
-    def _release_table_lock(self, database: str, table: str) -> None:
-        """
-        释放表处理锁。
-        
-        Args:
-            database (str): 数据库名。
-            table (str): 表名。
-        """
-        key = f"{database}.{table}"
-
-        with self._lock:
-            if key in self._processing_tables:
-                self._processing_tables.remove(key)
-
-    @_retry_on_failure
     def _ensure_index(self, database: str, table: str, date_column: str) -> None:
         """
-        检查并为date_column自动创建索引（如果未存在）。
+        检查并为 date_column 自动创建索引（如果未存在）。
+        
         Args:
             database (str): 数据库名。
             table (str): 表名。
             date_column (str): 需要检查的日期列名。
         """
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 # 检查索引是否已存在
                 cursor.execute(
@@ -356,11 +311,9 @@ class MySQLDeduplicator:
                     try:
                         cursor.execute(f"CREATE INDEX `{safe_index_name}` ON `{database}`.`{table}` (`{date_column}`)")
                         conn.commit()
-                        logger.info('已自动为date_column创建索引', {"库": database, "表": table, "date_column": date_column, "索引名": safe_index_name})
+                        logger.debug('已自动为date_column创建索引', {"库": database, "表": table, "date_column": date_column, "索引名": safe_index_name})
                     except Exception as e:
                         logger.error('自动创建date_column索引失败', {"库": database, "表": table, "date_column": date_column, "异常": str(e)})
-                else:
-                    logger.debug('date_column已存在索引', {"库": database, "表": table, "date_column": date_column})
 
     def _row_generator(self, database, table, select_cols, select_where, batch_size=10000):
         """
@@ -377,7 +330,7 @@ class MySQLDeduplicator:
         offset = 0
         while True:
             sql = f"SELECT {select_cols} FROM `{database}`.`{table}` {select_where} LIMIT {batch_size} OFFSET {offset}"
-            with self._get_connection() as conn:
+            with self._conn_ctx() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
                     rows = cursor.fetchall()
@@ -388,85 +341,184 @@ class MySQLDeduplicator:
             if len(rows) < batch_size:
                 break
             offset += batch_size
-
-    def _get_all_dates(self, database: str, table: str, date_column: str) -> list:
+            
+    def _get_all_dates(self, database: str, table: str, date_column: str) -> List[str]:
         """
         获取表中所有不同的日期分区（按天）。
+        
         Args:
             database (str): 数据库名。
             table (str): 表名。
             date_column (str): 日期列名。
         Returns:
-            List: 所有不同的日期（字符串）。
+            List[str]: 所有不同的日期（字符串）。
         """
         sql = f"SELECT DISTINCT `{date_column}` FROM `{database}`.`{table}` ORDER BY `{date_column}` ASC"
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(sql)
                 return [row[date_column] for row in cursor.fetchall() if row[date_column] is not None]
 
     def _deduplicate_table(
-            self,
-            database: str,
-            table: str,
-            columns: Optional[List[str]] = None,
-            dry_run: bool = False,
-            use_python_dedup: bool = False,
-            dedup_start_date: Optional[str] = None,
-            dedup_end_date: Optional[str] = None,
-            lock_table: bool = True
+        self,
+        database: str,
+        table: str,
+        columns: Optional[List[str]] = None,
+        dry_run: bool = False,
+        use_python_dedup: bool = False,
+        date_val: Optional[str] = None,
+        lock_table: bool = True
     ) -> Tuple[int, int]:
         """
-        执行单表去重。
-        支持按天分批处理（如果表包含date_column），否则全表去重。
-        如果date_column在exclude_columns中，直接跳过该表。
-        优化：分批删除时用主键、避免重复建/删临时表、并发处理每天。
+        执行单表单天去重。只处理 date_val 这一天的数据（如果有 date_column），否则全表。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            columns (Optional[List[str]]): 指定去重列。
+            dry_run (bool): 是否为模拟运行。
+            use_python_dedup (bool): 是否用 Python 方式去重。
+            date_val (Optional[str]): 指定处理的日期（如有 date_column）。
+            lock_table (bool): 是否加表级锁。
+        Returns:
+            Tuple[int, int]: (重复组数, 实际删除行数)
         """
         if lock_table and not self._acquire_table_lock(database, table):
             return (0, 0)
         temp_table = None
         try:
-            # 获取实际列名
             all_columns = self._get_table_columns(database, table)
             all_columns_lower = [col.lower() for col in all_columns]
             exclude_columns_lower = [col.lower() for col in getattr(self, 'exclude_columns', [])]
             time_col = self.date_column
             time_col_lower = time_col.lower() if time_col else None
-            # 1. 跳过date_column在exclude_columns的情况
             if time_col_lower and time_col_lower in exclude_columns_lower:
                 logger.warning('date_column在exclude_columns中，跳过该表', {"库": database, "表": table, "date_column": time_col, "exclude_columns": self.exclude_columns})
                 return (0, 0)
-            # 2. 判断表是否包含date_column
             has_time_col = time_col_lower in all_columns_lower if time_col_lower else False
-            # 如果包含date_column，自动检查并创建索引
-            if has_time_col and dedup_start_date is None and dedup_end_date is None:
+
+            # 只要有date_column，始终分天处理（本函数只处理一天）
+            if has_time_col and date_val is not None:
                 self._ensure_index(database, table, time_col)
-                # 按天分区多线程处理
-                all_dates = self._get_all_dates(database, table, time_col)
-                total_dup = 0
-                total_del = 0
-                def process_date(date_val):
-                    try:
-                        logger.debug('按天分区去重', {"库": database, "表": table, "日期": date_val})
-                        dup_count, affected_rows = self._deduplicate_table(
-                            database, table, columns, dry_run, use_python_dedup,
-                            dedup_start_date=date_val, dedup_end_date=date_val,
-                            lock_table=False
-                        )
-                        return (dup_count, affected_rows, date_val, None)
-                    except Exception as e:
-                        logger.error('分区去重异常', {"库": database, "表": table, "日期": date_val, "异常": str(e), "func": sys._getframe().f_code.co_name})
-                        return (0, 0, date_val, str(e))
-                with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
-                    future_to_date = {executor.submit(process_date, date_val): date_val for date_val in all_dates}
-                    for future in concurrent.futures.as_completed(future_to_date):
-                        dup_count, affected_rows, date_val, err = future.result()
-                        if err:
-                            logger.warning('分区处理失败', {"库": database, "表": table, "日期": date_val, "异常": err, "func": sys._getframe().f_code.co_name})
-                        total_dup += dup_count
-                        total_del += affected_rows
-                return (total_dup, total_del)
-            # 获取去重列
+                # 获取去重列
+                use_columns = columns or all_columns
+                use_columns = [col for col in use_columns if col.lower() in all_columns_lower and col.lower() not in exclude_columns_lower]
+                invalid_columns = set([col for col in (columns or []) if col.lower() not in all_columns_lower])
+                if invalid_columns:
+                    logger.warning('不存在的列', {"库": database, "表": table, "不存在以下列": invalid_columns, 'func': sys._getframe().f_code.co_name})
+                if not use_columns:
+                    logger.error('没有有效的去重列', {"库": database, "表": table, "func": sys._getframe().f_code.co_name})
+                    return (0, 0)
+                pk = self.primary_key
+                pk_real = next((c for c in all_columns if c.lower() == pk.lower()), pk)
+                where_sql = f"t.`{time_col}` = '{date_val}'"
+                # 获取原始数据总量（只统计当天数据）
+                with self._conn_ctx() as conn:
+                    with conn.cursor() as cursor:
+                        count_where = f"WHERE `{time_col}` = '{date_val}'"
+                        count_sql = f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}` {count_where}"
+                        logger.debug('执行SQL', {'sql': count_sql})
+                        cursor.execute(count_sql)
+                        total_count_row = cursor.fetchone()
+                        total_count = total_count_row['cnt'] if total_count_row and 'cnt' in total_count_row else 0
+                logger.debug('执行', {"库": database, "表": table, "开始处理数据量": total_count, 'func': sys._getframe().f_code.co_name, "数据日期": date_val})
+                column_list = ', '.join([f'`{col}`' for col in use_columns])
+
+                # 用Python查找重复
+                if use_python_dedup:
+                    select_cols = f'`{pk_real}`,' + ','.join([f'`{col}`' for col in use_columns])
+                    select_where = f"WHERE `{time_col}` = '{date_val}'"
+                    grouped = defaultdict(list)
+                    for row in self._row_generator(database, table, select_cols, select_where, self.batch_size):
+                        key = tuple(row[col] for col in use_columns)
+                        grouped[key].append(row[pk_real])
+                    dup_count = 0
+                    del_ids = []
+                    for ids in grouped.values():
+                        if len(ids) > 1:
+                            dup_count += 1
+                            del_ids.extend(ids[1:])
+                    affected_rows = 0
+                    if not dry_run and del_ids:
+                        with self._conn_ctx() as conn:
+                            with conn.cursor() as cursor:
+                                for i in range(0, len(del_ids), self.batch_size):
+                                    batch_ids = del_ids[i:i+self.batch_size]
+                                    del_ids_str = ','.join([str(i) for i in batch_ids])
+                                    delete_sql = f"DELETE FROM `{database}`.`{table}` WHERE `{pk_real}` IN ({del_ids_str})"
+                                    cursor.execute(delete_sql)
+                                    batch_deleted = cursor.rowcount
+                                    affected_rows += batch_deleted
+                                    conn.commit()
+                    logger.debug('去重完成', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "Python", "数据处理": self.duplicate_keep_mode, "数据日期": date_val})
+                    return (dup_count, affected_rows)
+                # SQL方式查找重复
+                temp_table = self._make_temp_table_name(table)
+                drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
+                create_temp_where = f"WHERE `{time_col}` = '{date_val}'"
+                create_temp_sql = f"""
+                CREATE TABLE `{database}`.`{temp_table}` AS
+                SELECT MIN(`{pk_real}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
+                FROM `{database}`.`{table}`
+                {create_temp_where}
+                GROUP BY {column_list}
+                HAVING COUNT(*) > 1
+                """
+                with self._conn_ctx() as conn:
+                    with conn.cursor() as cursor:
+                        logger.debug('创建临时表SQL', {'sql': create_temp_sql})
+                        cursor.execute(create_temp_sql)
+                        cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`")
+                        dup_count_row = cursor.fetchone()
+                        dup_count = dup_count_row['cnt'] if dup_count_row and 'cnt' in dup_count_row else 0
+                        if dup_count == 0:
+                            logger.debug('没有重复数据', {"库": database, "表": table, "数据量": total_count, "数据日期": date_val})
+                            cursor.execute(drop_temp_sql)
+                            conn.commit()
+                            return (0, 0)
+                        affected_rows = 0
+                        if not dry_run:
+                            while True:
+                                where_clauses = []
+                                if self.duplicate_keep_mode == 'keep_one':
+                                    where_clauses.append(f"t.`{pk_real}` <> tmp.`min_id`")
+                                if where_sql.strip():
+                                    where_clauses.append(where_sql.strip())
+                                where_full = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
+                                find_dup_ids_sql = f"""
+                                SELECT t.`{pk_real}` as del_id
+                                FROM `{database}`.`{table}` t
+                                JOIN `{database}`.`{temp_table}` tmp
+                                ON {' AND '.join([f't.`{col}` <=> tmp.`{col}`' for col in use_columns])}
+                                {where_full}
+                                LIMIT {self.batch_size}
+                                """
+                                logger.debug('查找待删除重复id SQL', {'sql': find_dup_ids_sql})
+                                cursor.execute(find_dup_ids_sql)
+                                del_ids = [row['del_id'] for row in cursor.fetchall()]
+                                if not del_ids:
+                                    break
+                                del_ids_str = ','.join([str(i) for i in del_ids])
+                                delete_sql = f"DELETE FROM `{database}`.`{table}` WHERE `{pk_real}` IN ({del_ids_str})"
+                                logger.debug('按id批量删除SQL', {'sql': delete_sql, 'ids': del_ids})
+                                cursor.execute(delete_sql)
+                                batch_deleted = cursor.rowcount
+                                affected_rows += batch_deleted
+                                conn.commit()
+                                if batch_deleted == 0:
+                                    logger.warning('检测到未能删除任何数据，强制跳出循环，防止假死', {"库": database, "表": table})
+                                    break
+                                if batch_deleted < self.batch_size:
+                                    break
+                            logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "SQL", "数据处理": self.duplicate_keep_mode, "数据日期": date_val})
+                        else:
+                            logger.debug('dry_run模式，不执行删除', {"库": database, "表": table, "重复组": dup_count})
+                            affected_rows = 0
+                        cursor.execute(drop_temp_sql)
+                        conn.commit()
+                        return (dup_count, affected_rows)
+            # 没有date_column，处理全表
+            # ...existing code for full-table deduplication (as before, but without recursion)...
             use_columns = columns or all_columns
             use_columns = [col for col in use_columns if col.lower() in all_columns_lower and col.lower() not in exclude_columns_lower]
             invalid_columns = set([col for col in (columns or []) if col.lower() not in all_columns_lower])
@@ -477,70 +529,53 @@ class MySQLDeduplicator:
                 return (0, 0)
             pk = self.primary_key
             pk_real = next((c for c in all_columns if c.lower() == pk.lower()), pk)
-            # 判断是否需要加日期区间条件
-            where_sql = ''
-            if has_time_col and dedup_start_date and dedup_end_date:
-                where_sql = f"t.`{time_col}` >= '{dedup_start_date}' AND t.`{time_col}` <= '{dedup_end_date}'"
-            # 获取原始数据总量（只统计区间内数据）
-            with self._get_connection() as conn:
+            # 获取原始数据总量
+            with self._conn_ctx() as conn:
                 with conn.cursor() as cursor:
-                    count_where = f"WHERE `{time_col}` >= '{dedup_start_date}' AND `{time_col}` <= '{dedup_end_date}'" if has_time_col and dedup_start_date and dedup_end_date else ''
-                    count_sql = f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}` {count_where}"
+                    count_sql = f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}`"
                     logger.debug('执行SQL', {'sql': count_sql})
                     cursor.execute(count_sql)
                     total_count_row = cursor.fetchone()
                     total_count = total_count_row['cnt'] if total_count_row and 'cnt' in total_count_row else 0
-            logger.info('执行', {"库": database, "表": table, "开始处理数据量": total_count, 'func': sys._getframe().f_code.co_name, "数据日期": dedup_end_date})
+            logger.debug('执行', {"库": database, "表": table, "开始处理数据量": total_count, 'func': sys._getframe().f_code.co_name})
             column_list = ', '.join([f'`{col}`' for col in use_columns])
-
-            # 用Python查找重复
             if use_python_dedup:
-                # 1. 拉取所有数据（生成器分批拉取）
                 select_cols = f'`{pk_real}`,' + ','.join([f'`{col}`' for col in use_columns])
-                select_where = f"WHERE `{time_col}` >= '{dedup_start_date}' AND `{time_col}` <= '{dedup_end_date}'" if has_time_col and dedup_start_date and dedup_end_date else ''
-                select_sql = f"SELECT {select_cols} FROM `{database}`.`{table}` {select_where}"
-                logger.debug('用Python查找重复，拉取数据SQL', {'sql': select_sql})
-                # 用生成器分批拉取
+                select_where = ''
                 grouped = defaultdict(list)
                 for row in self._row_generator(database, table, select_cols, select_where, self.batch_size):
                     key = tuple(row[col] for col in use_columns)
                     grouped[key].append(row[pk_real])
-                # 2. 统计重复组和待删除id
                 dup_count = 0
                 del_ids = []
                 for ids in grouped.values():
                     if len(ids) > 1:
                         dup_count += 1
-                        del_ids.extend(ids[1:])  # 只保留第一个
+                        del_ids.extend(ids[1:])
                 affected_rows = 0
                 if not dry_run and del_ids:
-                    with self._get_connection() as conn:
+                    with self._conn_ctx() as conn:
                         with conn.cursor() as cursor:
                             for i in range(0, len(del_ids), self.batch_size):
-                                batch = del_ids[i:i+self.batch_size]
-                                del_ids_str = ','.join([str(i) for i in batch])
+                                batch_ids = del_ids[i:i+self.batch_size]
+                                del_ids_str = ','.join([str(i) for i in batch_ids])
                                 delete_sql = f"DELETE FROM `{database}`.`{table}` WHERE `{pk_real}` IN ({del_ids_str})"
-                                logger.debug('用Python分批删除SQL', {'sql': delete_sql, 'ids': batch})
                                 cursor.execute(delete_sql)
                                 batch_deleted = cursor.rowcount
                                 affected_rows += batch_deleted
                                 conn.commit()
-                logger.info('去重完成', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "Python", "数据处理": self.duplicate_keep_mode, "数据日期": dedup_end_date})
+                logger.debug('去重完成', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "Python", "数据处理": self.duplicate_keep_mode})
                 return (dup_count, affected_rows)
-            # SQL方式查找重复
             temp_table = self._make_temp_table_name(table)
             drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
-            # 创建临时表时加where条件
-            create_temp_where = f"WHERE `{time_col}` >= '{dedup_start_date}' AND `{time_col}` <= '{dedup_end_date}'" if has_time_col and dedup_start_date and dedup_end_date else ''
             create_temp_sql = f"""
             CREATE TABLE `{database}`.`{temp_table}` AS
             SELECT MIN(`{pk_real}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
             FROM `{database}`.`{table}`
-            {create_temp_where}
             GROUP BY {column_list}
             HAVING COUNT(*) > 1
             """
-            with self._get_connection() as conn:
+            with self._conn_ctx() as conn:
                 with conn.cursor() as cursor:
                     logger.debug('创建临时表SQL', {'sql': create_temp_sql})
                     cursor.execute(create_temp_sql)
@@ -548,7 +583,7 @@ class MySQLDeduplicator:
                     dup_count_row = cursor.fetchone()
                     dup_count = dup_count_row['cnt'] if dup_count_row and 'cnt' in dup_count_row else 0
                     if dup_count == 0:
-                        logger.info('没有重复数据', {"库": database, "表": table, "数据量": total_count, "数据日期": dedup_end_date})
+                        logger.info('没有重复数据', {"库": database, "表": table, "数据量": total_count})
                         cursor.execute(drop_temp_sql)
                         conn.commit()
                         return (0, 0)
@@ -558,8 +593,6 @@ class MySQLDeduplicator:
                             where_clauses = []
                             if self.duplicate_keep_mode == 'keep_one':
                                 where_clauses.append(f"t.`{pk_real}` <> tmp.`min_id`")
-                            if where_sql.strip():
-                                where_clauses.append(where_sql.strip())
                             where_full = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
                             find_dup_ids_sql = f"""
                             SELECT t.`{pk_real}` as del_id
@@ -586,7 +619,7 @@ class MySQLDeduplicator:
                                 break
                             if batch_deleted < self.batch_size:
                                 break
-                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "SQL", "数据处理": self.duplicate_keep_mode, "数据日期": dedup_end_date})
+                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "SQL", "数据处理": self.duplicate_keep_mode})
                     else:
                         logger.debug('dry_run模式，不执行删除', {"库": database, "表": table, "重复组": dup_count})
                         affected_rows = 0
@@ -595,10 +628,9 @@ class MySQLDeduplicator:
                     return (dup_count, affected_rows)
         except Exception as e:
             logger.error('异常', {"库": database, "表": table, "异常": str(e), 'func': sys._getframe().f_code.co_name, 'traceback': repr(e)})
-            # 异常时也要清理临时表
             if temp_table:
                 try:
-                    with self._get_connection() as conn:
+                    with self._conn_ctx() as conn:
                         with conn.cursor() as cursor:
                             drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
                             cursor.execute(drop_temp_sql)
@@ -611,26 +643,26 @@ class MySQLDeduplicator:
                 self._release_table_lock(database, table)
 
     def deduplicate_table(
-            self,
-            database: str,
-            table: str,
-            columns: Optional[List[str]] = None,
-            dry_run: bool = False,
-            reorder_id: bool = False,
-            use_python_dedup: bool = True
+        self,
+        database: str,
+        table: str,
+        columns: Optional[List[str]] = None,
+        dry_run: bool = False,
+        reorder_id: bool = False,
+        use_python_dedup: bool = True
     ) -> Tuple[int, int]:
         """
-        对指定表进行去重。
+        对指定表进行去重。始终按天分区（如有 date_column），否则全表。
         
         Args:
             database (str): 数据库名。
             table (str): 表名。
-            columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
-            dry_run (bool): 是否为模拟运行（只统计不实际删除）。
-            reorder_id (bool): 去重后是否重排id。
-            use_python_dedup (bool): 是否用Python查找重复id。
+            columns (Optional[List[str]]): 指定去重列。
+            dry_run (bool): 是否为模拟运行。
+            reorder_id (bool): 去重后是否自动重排 id 列。
+            use_python_dedup (bool): 是否用 Python 方式去重。
         Returns:
-            Tuple[int, int]: (重复组数, 实际删除行数)。
+            Tuple[int, int]: (重复组数, 实际删除行数)
         """
         if database.lower() in self.exclude_tables and table.lower() in self.exclude_tables[database.lower()]:
             logger.info('表被排除', {"库": database, "表": table, "操作": "跳过"})
@@ -639,10 +671,76 @@ class MySQLDeduplicator:
             if not self._check_table_exists(database, table):
                 logger.warning('表不存在', {"库": database, "表": table, "warning": "跳过"})
                 return (0, 0)
-            logger.info('单表开始', {"库": database, "表": table, "参数": {"指定去重列": columns, "模拟运行": dry_run, '排除列': self.exclude_columns, 'use_python_dedup': use_python_dedup}})
-            result = self._deduplicate_table(database, table, columns, dry_run, use_python_dedup)
-            logger.info('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result})
-            # 自动重排id列（仅当有实际删除时且reorder_id为True）
+            logger.info('单表开始', {
+                "库": database, 
+                "表": table, 
+                # "参数": {
+                #     "指定去重列": columns, 
+                #     "去重方式": "Python" if use_python_dedup else "SQL", 
+                #     "数据处理": self.duplicate_keep_mode,
+                #     "模拟运行": dry_run, 
+                #     '排除列': self.exclude_columns,
+                #     },
+                })
+            all_columns = self._get_table_columns(database, table)
+            all_columns_lower = [col.lower() for col in all_columns]
+            time_col = self.date_column
+            time_col_lower = time_col.lower() if time_col else None
+            has_time_col = time_col_lower in all_columns_lower if time_col_lower else False
+            if has_time_col:
+                self._ensure_index(database, table, time_col)
+                all_dates = self._get_all_dates(database, table, time_col)
+                # 按date_range/recent_month筛选日期
+                start_date = self._dedup_start_date
+                end_date = self._dedup_end_date
+                if start_date and end_date:
+                    all_dates = [d for d in all_dates if str(start_date) <= str(d) <= str(end_date)]
+                if not all_dates:
+                    logger.info('无可处理日期', {"库": database, "表": table})
+                    return (0, 0)
+                total_dup = 0
+                total_del = 0
+                def process_date(date_val):
+                    try:
+                        logger.debug('按天分区去重', {"库": database, "表": table, "日期": date_val})
+                        dup_count, affected_rows = self._deduplicate_table(
+                            database, table, columns, dry_run, use_python_dedup,
+                            date_val=date_val, lock_table=False
+                        )
+                        return (dup_count, affected_rows, date_val, None)
+                    except Exception as e:
+                        logger.error('分区去重异常', {"库": database, "表": table, "日期": date_val, "异常": str(e), "func": sys._getframe().f_code.co_name})
+                        return (0, 0, date_val, str(e))
+                if self.max_workers > 1:
+                    with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                        future_to_date = {executor.submit(process_date, date_val): date_val for date_val in all_dates}
+                        for future in concurrent.futures.as_completed(future_to_date):
+                            dup_count, affected_rows, date_val, err = future.result()
+                            if err:
+                                logger.warning('分区处理失败', {"库": database, "表": table, "日期": date_val, "异常": err, "func": sys._getframe().f_code.co_name})
+                            total_dup += dup_count
+                            total_del += affected_rows
+                else:
+                    for date_val in all_dates:
+                        dup_count, affected_rows, _, err = process_date(date_val)
+                        if err:
+                            logger.warning('分区处理失败', {"库": database, "表": table, "日期": date_val, "异常": err, "func": sys._getframe().f_code.co_name})
+                        total_dup += dup_count
+                        total_del += affected_rows
+                logger.debug('单表完成', {"库": database, "表": table, "结果[重复, 删除]": (total_dup, total_del), '日期范围': f"{start_date} - {end_date}"})
+                # 自动重排id列（仅当有实际删除时且reorder_id为True）
+                if reorder_id and total_del > 0:
+                    try:
+                        reorder_ok = self.reorder_id_column(database, table, id_column=self.primary_key, dry_run=dry_run)
+                        logger.info('自动重排id列完成', {"库": database, "表": table, "结果": reorder_ok})
+                    except Exception as e:
+                        logger.error('自动重排id列异常', {"库": database, "表": table, "异常": str(e)})
+                if affected_rows > 0:
+                    logger.info('单表完成(仅显示有删除的结果)', {"库": database, "表": table, "重复组": total_dup, "实际删除": total_del})
+                return (total_dup, total_del)
+            # 没有date_column，直接全表去重
+            result = self._deduplicate_table(database, table, columns, dry_run, use_python_dedup, date_val=None)
+            logger.debug('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result, '日期范围': '全表'})
             dup_count, affected_rows = result
             if reorder_id and affected_rows > 0:
                 try:
@@ -650,34 +748,36 @@ class MySQLDeduplicator:
                     logger.info('自动重排id列完成', {"库": database, "表": table, "结果": reorder_ok})
                 except Exception as e:
                     logger.error('自动重排id列异常', {"库": database, "表": table, "异常": str(e)})
+            if affected_rows > 0:
+                logger.info('单表完成(仅显示有删除的结果)', {"库": database, "表": table, "重复组": dup_count, "实际删除": affected_rows})
             return result
         except Exception as e:
             logger.error('发生全局错误', {"库": database, "表": table, 'func': sys._getframe().f_code.co_name, "发生全局错误": str(e)})
             return (0, 0)
 
     def deduplicate_database(
-            self,
-            database: str,
-            tables: Optional[List[str]] = None,
-            columns_map: Optional[Dict[str, List[str]]] = None,
-            dry_run: bool = False,
-            parallel: bool = False,
-            reorder_id: bool = False,
-            use_python_dedup: bool = True
+        self,
+        database: str,
+        tables: Optional[List[str]] = None,
+        columns_map: Optional[Dict[str, List[str]]] = None,
+        dry_run: bool = False,
+        parallel: bool = False,
+        reorder_id: bool = False,
+        use_python_dedup: bool = True
     ) -> Dict[str, Tuple[int, int]]:
         """
-        对指定数据库的所有表进行去重。
+        对指定数据库的所有表进行去重。调用 deduplicate_table，自动适配分天。
         
         Args:
             database (str): 数据库名。
-            tables (Optional[List[str]]): 要处理的表列表（为None时处理所有表）。
-            columns_map (Optional[Dict[str, List[str]]]): 各表使用的去重列 {表名: [列名]}。
+            tables (Optional[List[str]]): 指定表名列表。
+            columns_map (Optional[Dict[str, List[str]]]): 每个表的去重列映射。
             dry_run (bool): 是否为模拟运行。
-            parallel (bool): 是否并行处理。
-            reorder_id (bool): 去重后是否重排id。
-            use_python_dedup (bool): 是否用Python查找重复id。
+            parallel (bool): 是否并行处理表。
+            reorder_id (bool): 去重后是否自动重排 id 列。
+            use_python_dedup (bool): 是否用 Python 方式去重。
         Returns:
-            Dict[str, Tuple[int, int]]: {表名: (重复组数, 实际删除行数)}。
+            Dict[str, Tuple[int, int]]: {表名: (重复组数, 实际删除行数)}
         """
         results = {}
         try:
@@ -693,8 +793,6 @@ class MySQLDeduplicator:
                 return results
             logger.info('库统计', {"库": database, "表数量": len(target_tables), "表列表": target_tables})
             if parallel and self.max_workers > 1:
-                logger.debug('并行处理表', {'库': database, 'max_workers': self.max_workers})
-                # 使用线程池并行处理
                 with concurrent.futures.ThreadPoolExecutor(
                         max_workers=self.max_workers
                 ) as executor:
@@ -715,8 +813,6 @@ class MySQLDeduplicator:
                             logger.error('异常', {"库": database, "表": table, "error": str(e), 'traceback': repr(e)})
                             results[table] = (0, 0)
             else:
-                logger.debug('串行处理表', {'库': database})
-                # 串行处理
                 for table in target_tables:
                     columns = columns_map.get(table) if columns_map else None
                     dup_count, affected_rows = self.deduplicate_table(
@@ -725,35 +821,39 @@ class MySQLDeduplicator:
                     results[table] = (dup_count, affected_rows)
             total_dup = sum(r[0] for r in results.values())
             total_del = sum(r[1] for r in results.values())
-            logger.info('单库完成', {"库": database, "重复组": total_dup, "总删除行": total_del, "详细结果": results})
+            logger.debug('库完成', {"库": database, "重复组": total_dup, "总删除行": total_del, "详细结果": results})
+            # 只显示有删除的详细结果
+            if total_del > 0:
+                filtered_results = {tbl: res for tbl, res in results.items() if res[1] > 0}
+                logger.info('库完成(仅显示有删除的结果)', {"库": database, "重复组": total_dup, "总删除行": total_del, "详细结果": filtered_results})
             return results
         except Exception as e:
             logger.error('发生全局错误', {"库": database, 'func': sys._getframe().f_code.co_name, "error": str(e), 'traceback': repr(e)})
             return results
 
     def deduplicate_all(
-            self,
-            databases: Optional[List[str]] = None,
-            tables_map: Optional[Dict[str, List[str]]] = None,
-            columns_map: Optional[Dict[str, Dict[str, List[str]]]] = None,
-            dry_run: bool = False,
-            parallel: bool = False,
-            reorder_id: bool = False,
-            use_python_dedup: bool = True
+        self,
+        databases: Optional[List[str]] = None,
+        tables_map: Optional[Dict[str, List[str]]] = None,
+        columns_map: Optional[Dict[str, Dict[str, List[str]]]] = None,
+        dry_run: bool = False,
+        parallel: bool = False,
+        reorder_id: bool = False,
+        use_python_dedup: bool = True
     ) -> Dict[str, Dict[str, Tuple[int, int]]]:
         """
-        对所有数据库进行去重。
+        对所有数据库进行去重。调用 deduplicate_database，自动适配分天。
         
         Args:
-            databases (Optional[List[str]]): 要处理的数据库列表。如果为 None，则处理所有非系统数据库。
-            tables_map (Optional[Dict[str, List[str]]]): 指定每个数据库要处理的表，格式为 {数据库名: [表名, ...]}。如果为 None，则处理所有表。
-            columns_map (Optional[Dict[str, Dict[str, List[str]]]]): 指定每个表去重时使用的列，格式为 {数据库名: {表名: [列名, ...]}}。如果为 None，则使用所有列。
-            dry_run (bool): 是否为模拟运行模式。为 True 时只统计重复行数，不实际删除。
-            parallel (bool): 是否并行处理多个数据库。为 True 时使用线程池并发处理。
-            reorder_id (bool): 去重后是否重排id。
-            use_python_dedup (bool): 是否用Python查找重复id。
+            databases (Optional[List[str]]): 指定数据库名列表。
+            tables_map (Optional[Dict[str, List[str]]]): 每个库的表名映射。
+            columns_map (Optional[Dict[str, Dict[str, List[str]]]]): 每个库每个表的去重列映射。
+            dry_run (bool): 是否为模拟运行。
+            parallel (bool): 是否并行处理库。
+            reorder_id (bool): 去重后是否自动重排 id 列。
+            use_python_dedup (bool): 是否用 Python 方式去重。
         Returns:
-            Dict[str, Dict[str, Tuple[int, int]]]: 嵌套字典，格式为 {数据库名: {表名: (重复组数, 实际删除行数)}}。
+            Dict[str, Dict[str, Tuple[int, int]]]: {库: {表: (重复组数, 实际删除行数)}}
         """
         all_results: Dict[str, Dict[str, Tuple[int, int]]] = defaultdict(dict)
         try:
@@ -763,9 +863,19 @@ class MySQLDeduplicator:
             if not target_dbs:
                 logger.warning('没有可处理的数据库')
                 return all_results
-            logger.info('全局开始', {"数据库数量": len(target_dbs), "数据库列表": target_dbs, "参数": {"模拟运行": dry_run, "并行处理": parallel, '排除列': self.exclude_columns, 'use_python_dedup': use_python_dedup}})
-            if parallel and self.max_workers > 1:
-                # 使用线程池并行处理多个数据库
+            logger.info('全局开始', {
+                "数据库数量": len(target_dbs), 
+                "数据库列表": target_dbs, 
+                "参数": {
+                    "模拟运行": dry_run, 
+                    "并行处理": parallel, 
+                    '排除列': self.exclude_columns, 
+                    '重排id': reorder_id, 
+                    'use_python_dedup': use_python_dedup
+                    },
+                })
+            # 如果parallel=True且库数量大于1，则只在外层并发，内层串行
+            if parallel and self.max_workers > 1 and len(target_dbs) > 1:
                 with concurrent.futures.ThreadPoolExecutor(
                         max_workers=self.max_workers
                 ) as executor:
@@ -773,6 +883,7 @@ class MySQLDeduplicator:
                     for db in target_dbs:
                         tables = tables_map.get(db) if tables_map else None
                         db_columns_map = columns_map.get(db) if columns_map else None
+                        # 内层强制串行
                         futures[executor.submit(
                             self.deduplicate_database,
                             db, tables, db_columns_map, dry_run, False, reorder_id, use_python_dedup
@@ -786,7 +897,6 @@ class MySQLDeduplicator:
                             logger.error('异常', {"库": db, "error": str(e), 'traceback': repr(e)})
                             all_results[db] = {}
             else:
-                # 串行处理数据库
                 for db in target_dbs:
                     tables = tables_map.get(db) if tables_map else None
                     db_columns_map = columns_map.get(db) if columns_map else None
@@ -802,13 +912,42 @@ class MySQLDeduplicator:
                 r[1] for db in all_results.values()
                 for r in db.values()
             )
-            logger.info('全局完成', {"总重复组": total_dup, "总删除行": total_del, "详细结果": dict(all_results)})
+            logger.debug('全局完成', {
+                "总重复组": total_dup, 
+                "总删除行": total_del, 
+                "参数": {
+                    "模拟运行": dry_run, 
+                    "并行处理": parallel, 
+                    '排除列': self.exclude_columns, 
+                    '重排id': reorder_id, 
+                    'use_python_dedup': use_python_dedup
+                    }, 
+                "详细结果": dict(all_results)
+                })
+            # 只显示有删除的详细结果
+            if total_del > 0:
+                filtered_results = {
+                    db: {tbl: res for tbl, res in tbls.items() if res[1] > 0}
+                    for db, tbls in all_results.items()
+                }
+                filtered_results = {db: tbls for db, tbls in filtered_results.items() if tbls}
+                logger.info('全局完成(仅显示有删除的结果)', {
+                    "总重复组": total_dup,
+                    "总删除行": total_del,
+                    "参数": {
+                        "模拟运行": dry_run,
+                        "并行处理": parallel,
+                        '排除列': self.exclude_columns,
+                        '重排id': reorder_id,
+                        'use_python_dedup': use_python_dedup
+                    },
+                    "详细结果": filtered_results
+                })
             return all_results
         except Exception as e:
             logger.error('异常', {"error": str(e), 'traceback': repr(e)})
             return all_results
 
-    @_retry_on_failure
     def _check_database_exists(self, database: str) -> bool:
         """
         检查数据库是否存在。
@@ -819,13 +958,11 @@ class MySQLDeduplicator:
             bool: 数据库是否存在。
         """
         sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
-
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(sql, (database,))
                 return bool(cursor.fetchone())
 
-    @_retry_on_failure
     def _check_table_exists(self, database: str, table: str) -> bool:
         """
         检查表是否存在。
@@ -841,12 +978,35 @@ class MySQLDeduplicator:
         FROM INFORMATION_SCHEMA.TABLES 
         WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
         """
-
-        with self._get_connection() as conn:
+        with self._conn_ctx() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(sql, (database, table))
                 return bool(cursor.fetchone())
 
+    def _get_table_info(self, database: str, table: str, id_column: str = None):
+        """
+        获取表的所有列名、主键列名列表、指定id列是否为主键。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            id_column (str): id列名，默认使用self.primary_key。
+        Returns:
+            Tuple[List[str], List[str], bool]: (所有列名, 主键列名, id列是否为主键)
+        """
+        id_column = id_column or self.primary_key
+        with self._conn_ctx() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute("""
+                    SELECT COLUMN_NAME, COLUMN_KEY
+                    FROM INFORMATION_SCHEMA.COLUMNS
+                    WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s
+                """, (database, table))
+                columns_info = cursor.fetchall()
+                columns = [row['COLUMN_NAME'] for row in columns_info]
+                pk_cols = [row['COLUMN_NAME'] for row in columns_info if row['COLUMN_KEY'] == 'PRI']
+                id_is_pk = any(row['COLUMN_NAME'].lower() == id_column.lower() and row['COLUMN_KEY'] in ('PRI', 'UNI') for row in columns_info)
+        return columns, pk_cols, id_is_pk
+
     def close(self) -> None:
         """
         关闭连接池。
@@ -895,15 +1055,16 @@ class MySQLDeduplicator:
         auto_drop_backup: bool = True
     ) -> Any:
         """
-        安全重排指定表或指定库下所有表的id列为顺序自增（1,2,3...）。
+        安全重排指定表或指定库下所有表的 id 列为顺序自增（1,2,3...）。
+        
         Args:
-            database (str): 数据库名
-            table (Optional[str]): 表名，None时批量处理该库所有表
-            id_column (str): id列名，默认"id"
-            dry_run (bool): 是否为模拟运行
-            auto_drop_backup (bool): 校验通过后自动删除备份表
+            database (str): 数据库名。
+            table (Optional[str]): 表名，None 时批量处理该库所有表。
+            id_column (str): id 列名，默认 "id"。
+            dry_run (bool): 是否为模拟运行。
+            auto_drop_backup (bool): 校验通过后自动删除备份表。
         Returns:
-            bool 或 dict: 单表时bool，批量时{表名: bool}
+            bool 或 dict: 单表时 bool，批量时 {表名: bool}
         """
         if not table:
             # 批量模式，对库下所有表执行
@@ -942,7 +1103,7 @@ class MySQLDeduplicator:
                 logger.warning('主键不是单列id，跳过id重排', {"库": database, "表": table, "主键列": pk_cols})
                 return False
             # 检查外键约束
-            with self._get_connection() as conn:
+            with self._conn_ctx() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute("""
                         SELECT * FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
@@ -952,7 +1113,7 @@ class MySQLDeduplicator:
                         logger.warning('表存在外键约束，跳过id重排', {"库": database, "表": table})
                         return False
             # 获取表结构
-            with self._get_connection() as conn:
+            with self._conn_ctx() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(f"SHOW CREATE TABLE {table_quoted}")
                     create_table_sql = cursor.fetchone()['Create Table']
@@ -965,7 +1126,7 @@ class MySQLDeduplicator:
             backup_table = self._make_backup_table_name(table)
             backup_table_quoted = f"`{database}`.`{backup_table}`"
             try:
-                with self._get_connection() as conn:
+                with self._conn_ctx() as conn:
                     with conn.cursor() as cursor:
                         # 1. 创建临时表，结构同原表
                         try:
@@ -1026,7 +1187,7 @@ class MySQLDeduplicator:
                                 logger.error('回滚恢复原表失败', {"库": database, "表": table, "异常": str(e)})
                             return False
                         logger.info('id重排成功且数据量一致', {"库": database, "表": table, "新表": new_cnt, "备份表": old_cnt, "备份表名": backup_table})
-                        # 5. 可选：自动删除备份表
+                        # 5. 自动删除备份表
                         if auto_drop_backup:
                             try:
                                 cursor.execute(f"DROP TABLE {backup_table_quoted}")
@@ -1037,7 +1198,7 @@ class MySQLDeduplicator:
             except Exception as e:
                 logger.error('id重排异常，准备回滚', {"库": database, "表": table, "异常": str(e)})
                 # 回滚：如临时表存在则删掉，恢复原表结构
-                with self._get_connection() as conn:
+                with self._conn_ctx() as conn:
                     with conn.cursor() as cursor:
                         try:
                             cursor.execute(f"DROP TABLE IF EXISTS {temp_table_quoted}")
@@ -1045,7 +1206,7 @@ class MySQLDeduplicator:
                             logger.error('回滚时删除临时表失败', {"库": database, "表": table, "异常": str(drop_e)})
                         # 恢复原表（如备份表存在）
                         try:
-                            with self._get_connection() as conn2:
+                            with self._conn_ctx() as conn2:
                                 with conn2.cursor() as cursor2:
                                     if self._check_table_exists(database, backup_table):
                                         cursor2.execute(f"DROP TABLE IF EXISTS {table_quoted}")
@@ -1057,6 +1218,41 @@ class MySQLDeduplicator:
         finally:
             self._release_table_lock(database, table)
 
+    def _acquire_table_lock(self, database: str, table: str, timeout: int = 60) -> bool:
+        """
+        获取表级锁，防止多线程/多进程并发操作同一张表。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            timeout (int): 等待锁的超时时间（秒）。
+        Returns:
+            bool: 是否成功获取锁。
+        """
+        key = f"{database.lower()}::{table.lower()}"
+        start_time = time.time()
+        while True:
+            with self._lock:
+                if key not in self._processing_tables:
+                    self._processing_tables.add(key)
+                    return True
+            if time.time() - start_time > timeout:
+                logger.warning('获取表级锁超时', {"库": database, "表": table, "timeout": timeout})
+                return False
+            time.sleep(0.2)
+
+    def _release_table_lock(self, database: str, table: str) -> None:
+        """
+        释放表级锁。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+        Returns:
+            None
+        """
+        key = f"{database.lower()}::{table.lower()}"
+        with self._lock:
+            self._processing_tables.discard(key)
+
     @staticmethod
     def _make_safe_table_name(base: str, prefix: str = '', suffix: str = '', max_length: int = 64) -> str:
         """
@@ -1077,30 +1273,6 @@ class MySQLDeduplicator:
             return (prefix + suffix)[:max_length]
         return f"{prefix}{base[:remain]}{suffix}"[:max_length]
 
-    def _get_table_info(self, database: str, table: str, id_column: str = None):
-        """
-        获取表的所有列名、主键列名列表、指定id列是否为主键。
-        Args:
-            database (str): 数据库名。
-            table (str): 表名。
-            id_column (str): id列名，默认使用self.primary_key。
-        Returns:
-            Tuple[List[str], List[str], bool]: (所有列名, 主键列名, id列是否为主键)
-        """
-        id_column = id_column or self.primary_key
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute("""
-                    SELECT COLUMN_NAME, COLUMN_KEY
-                    FROM INFORMATION_SCHEMA.COLUMNS
-                    WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s
-                """, (database, table))
-                columns_info = cursor.fetchall()
-                columns = [row['COLUMN_NAME'] for row in columns_info]
-                pk_cols = [row['COLUMN_NAME'] for row in columns_info if row['COLUMN_KEY'] == 'PRI']
-                id_is_pk = any(row['COLUMN_NAME'].lower() == id_column.lower() and row['COLUMN_KEY'] in ('PRI', 'UNI') for row in columns_info)
-        return columns, pk_cols, id_is_pk
-
     def _make_temp_table_name(self, base: str) -> str:
         """
         生成临时表名，带有 temp_ 前缀和 _dedup_ 进程线程后缀。
@@ -1122,26 +1294,35 @@ def main():
         password='pwd',
         host='localhost',
         port=3306,
-        # date_range=['2025-05-27', '2025-05-28'],
-        exclude_tables={'推广数据2': [
-            # '地域报表_城市_2025_04', 
-            # '地域报表_城市_2025_05', 
-            # '地域报表_城市_2025_06', 
-            '地域报表_城市_2025_04_copy1', 
-            '地域报表_城市_2025_05_copy1', 
-            '地域报表_城市_2025_06_copy1', 
-            '主体报表_2025_copy1'
-            ]}
+        max_workers= 2,
+        batch_size=1000,
+        skip_system_dbs=True,
+        max_retries=3,
+        retry_waiting_time=5,
+        # pool_size=30,
+        recent_month=1,
+        # date_range=['2025-06-09', '2025-06-10'],
+        date_column='日期',
+        exclude_databases=['测试库4'],
+        exclude_tables={
+            '推广数据2': [
+                '地域报表_城市_2025_04', 
+                # '地域报表_城市_2025_04_copy1', 
+            ],
+            "生意参谋3": [
+                "商品排行_2025",
+            ],
+        },
     )
 
     # 全库去重(单线程)
-    deduplicator.deduplicate_all(dry_run=True, parallel=True, reorder_id=True)
+    deduplicator.deduplicate_all(dry_run=False, parallel=True, reorder_id=True)
 
     # # 指定数据库去重(多线程)
-    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=False, reorder_id=True)
+    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=True, reorder_id=True)
 
     # # 指定表去重(使用特定列)
-    # deduplicator.deduplicate_table('my_db', 'my_table', columns=["name", "date"], dry_run=False, reorder_id=False)
+    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'data'], dry_run=False, reorder_id=True)
 
     # # 重排id列
     # deduplicator.reorder_id_column('my_db', 'my_table', 'id', dry_run=False, auto_drop_backup=True)

{mdbq-3.11.10.dist-info → mdbq-3.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.11.10
+Version: 3.12.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.11.10.dist-info → mdbq-3.12.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=L9HK2W1LgO8Zc5gpJgI1uJ5J0VRcUyMXHr1ZT-FeNOM,19
+mdbq/__version__.py,sha256=W8WVhYkHLU0SBDlL9Q6XQVTqIrzYjc1kFBZgqzS_NEI,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -8,7 +8,7 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=Crw6LwVo3I3IUbzIETu8f46Quza3CTCh-qYf4edbBPo,24139
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=w8etA5dAsY7g58bWU3SQt7n_OWnS9Y2TVh0D7m0MK9E,57961
+mdbq/mysql/deduplicator.py,sha256=KMJ_YyqAniaLVRqOHLgO92PgwknIDB-EgaOY7S6iMZ4,68599
 mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
 mdbq/mysql/uploader.py,sha256=8Px_W2bYOr1wQgMXMK0DggNiuE6a6Ul4BlJake8LSo8,64469
@@ -24,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
-mdbq-3.11.10.dist-info/METADATA,sha256=dVhkC84iq1GWtV6onfsLj18CwfGnIo1bXXDa-TXUU1E,365
-mdbq-3.11.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.11.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.11.10.dist-info/RECORD,,
+mdbq-3.12.0.dist-info/METADATA,sha256=Q6EyaC61H4okFva6YFV2a0Y3Iqun8L8mnpSkeVXcFdc,364
+mdbq-3.12.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.12.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.12.0.dist-info/RECORD,,