mdbq 3.11.8__py3-none-any.whl → 3.11.10__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.11.8'
+VERSION = '3.11.10'

mdbq/log/mylogger.py

@@ -247,7 +247,7 @@ class MyLogger:
                             if isinstance(log_data.get('message'), str):
                                 log_data['message'] = log_data['message'].replace(field, '***')
 
-                    return json.dumps(log_data, ensure_ascii=False)
+                    return json.dumps(log_data, ensure_ascii=False, default=str)
 
             formatter = StructuredFormatter()
 

mdbq/mysql/deduplicator.py

@@ -37,7 +37,7 @@ class MySQLDeduplicator:
 
     功能：
     1. 自动检测并删除MySQL数据库中的重复数据
-    2. 支持全库扫描或指定表处理
+    2. 支持全库扫描或指定表理
     3. 支持多线程/多进程安全处理
     4. 完善的错误处理和日志记录
 
@@ -114,7 +114,7 @@ class MySQLDeduplicator:
         )
 
         # 配置参数
-        self.max_workers = max(1, min(max_workers, 20))  # 限制最大线程数
+        self.max_workers = min(max(1, max_workers), pool_size)  # 限制最大线程数，不能超过连接池
         self.batch_size = batch_size
         self.skip_system_dbs = skip_system_dbs
         self.max_retries = max_retries
@@ -215,7 +215,7 @@ class MySQLDeduplicator:
             last_exception = None
             for attempt in range(self.max_retries + 1):
                 try:
-                    logger.debug(f'调用{func.__name__}，第{attempt+1}次尝试', {'args': args, 'kwargs': kwargs})
+                    logger.debug(f'调用{func.__name__}，第{attempt+1}次连接', {'args': args, 'kwargs': kwargs})
                     return func(self, *args, **kwargs)
                 except (pymysql.OperationalError, pymysql.InterfaceError) as e:
                     last_exception = e
@@ -269,7 +269,8 @@ class MySQLDeduplicator:
             with conn.cursor() as cursor:
                 cursor.execute(f"USE `{database}`")
                 cursor.execute(sql)
-                return [row[f'Tables_in_{database}'] for row in cursor.fetchall()]
+                # 严格过滤所有以'temp_'为前缀的表名（如temp_xxx、temp_xxx_dedup_...、temp_xxx_reorderid_...等）
+                return [row[f'Tables_in_{database}'] for row in cursor.fetchall() if not re.match(r'^temp_.*', row[f'Tables_in_{database}'])]
 
     @_retry_on_failure
     def _get_table_columns(self, database: str, table: str) -> List[str]:
@@ -328,129 +329,267 @@ class MySQLDeduplicator:
             if key in self._processing_tables:
                 self._processing_tables.remove(key)
 
+    @_retry_on_failure
+    def _ensure_index(self, database: str, table: str, date_column: str) -> None:
+        """
+        检查并为date_column自动创建索引（如果未存在）。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            date_column (str): 需要检查的日期列名。
+        """
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                # 检查索引是否已存在
+                cursor.execute(
+                    """
+                    SELECT COUNT(1) as idx_count FROM INFORMATION_SCHEMA.STATISTICS
+                    WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s
+                    """,
+                    (database, table, date_column)
+                )
+                idx_count = cursor.fetchone()['idx_count']
+                if idx_count == 0:
+                    # 自动创建索引
+                    index_name = f"idx_{date_column}"
+                    safe_index_name = self._make_safe_table_name(index_name, prefix='', suffix='', max_length=64)
+                    try:
+                        cursor.execute(f"CREATE INDEX `{safe_index_name}` ON `{database}`.`{table}` (`{date_column}`)")
+                        conn.commit()
+                        logger.info('已自动为date_column创建索引', {"库": database, "表": table, "date_column": date_column, "索引名": safe_index_name})
+                    except Exception as e:
+                        logger.error('自动创建date_column索引失败', {"库": database, "表": table, "date_column": date_column, "异常": str(e)})
+                else:
+                    logger.debug('date_column已存在索引', {"库": database, "表": table, "date_column": date_column})
+
+    def _row_generator(self, database, table, select_cols, select_where, batch_size=10000):
+        """
+        生成器：分批拉取表数据，避免一次性加载全部数据到内存。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            select_cols (str): 选择的列字符串。
+            select_where (str): where条件字符串。
+            batch_size (int): 每批拉取的行数。
+        Yields:
+            dict: 每行数据。
+        """
+        offset = 0
+        while True:
+            sql = f"SELECT {select_cols} FROM `{database}`.`{table}` {select_where} LIMIT {batch_size} OFFSET {offset}"
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                    rows = cursor.fetchall()
+                    if not rows:
+                        break
+                    for row in rows:
+                        yield row
+            if len(rows) < batch_size:
+                break
+            offset += batch_size
+
+    def _get_all_dates(self, database: str, table: str, date_column: str) -> list:
+        """
+        获取表中所有不同的日期分区（按天）。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            date_column (str): 日期列名。
+        Returns:
+            List: 所有不同的日期（字符串）。
+        """
+        sql = f"SELECT DISTINCT `{date_column}` FROM `{database}`.`{table}` ORDER BY `{date_column}` ASC"
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(sql)
+                return [row[date_column] for row in cursor.fetchall() if row[date_column] is not None]
+
     def _deduplicate_table(
             self,
             database: str,
             table: str,
             columns: Optional[List[str]] = None,
-            dry_run: bool = False
+            dry_run: bool = False,
+            use_python_dedup: bool = False,
+            dedup_start_date: Optional[str] = None,
+            dedup_end_date: Optional[str] = None,
+            lock_table: bool = True
     ) -> Tuple[int, int]:
         """
         执行单表去重。
-        
-        Args:
-            database (str): 数据库名。
-            table (str): 表名。
-            columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
-            dry_run (bool): 是否为模拟运行（只统计不实际删除）。
-        Returns:
-            Tuple[int, int]: (重复组数, 实际删除行数)。
+        支持按天分批处理（如果表包含date_column），否则全表去重。
+        如果date_column在exclude_columns中，直接跳过该表。
+        优化：分批删除时用主键、避免重复建/删临时表、并发处理每天。
         """
-        if not self._acquire_table_lock(database, table):
+        if lock_table and not self._acquire_table_lock(database, table):
             return (0, 0)
         temp_table = None
         try:
-            # 获取原始数据总量
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    logger.debug('执行SQL', {'sql': f'SELECT COUNT(*) as cnt FROM `{database}`.`{table}`'})
-                    cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}`")
-                    total_count_row = cursor.fetchone()
-                    total_count = total_count_row['cnt'] if total_count_row and 'cnt' in total_count_row else 0
-            logger.info('执行', {"库": database, "表": table, "开始处理数据量": total_count, 'func': sys._getframe().f_code.co_name})
             # 获取实际列名
             all_columns = self._get_table_columns(database, table)
-            logger.debug('获取表列', {'库': database, '表': table, 'all_columns': all_columns})
-            # 检查是否需要按时间范围过滤
-            use_time_filter = False
-            time_col = self.date_column
             all_columns_lower = [col.lower() for col in all_columns]
-            # 排除exclude_columns
             exclude_columns_lower = [col.lower() for col in getattr(self, 'exclude_columns', [])]
-            # 统一列名小写做判断
+            time_col = self.date_column
+            time_col_lower = time_col.lower() if time_col else None
+            # 1. 跳过date_column在exclude_columns的情况
+            if time_col_lower and time_col_lower in exclude_columns_lower:
+                logger.warning('date_column在exclude_columns中，跳过该表', {"库": database, "表": table, "date_column": time_col, "exclude_columns": self.exclude_columns})
+                return (0, 0)
+            # 2. 判断表是否包含date_column
+            has_time_col = time_col_lower in all_columns_lower if time_col_lower else False
+            # 如果包含date_column，自动检查并创建索引
+            if has_time_col and dedup_start_date is None and dedup_end_date is None:
+                self._ensure_index(database, table, time_col)
+                # 按天分区多线程处理
+                all_dates = self._get_all_dates(database, table, time_col)
+                total_dup = 0
+                total_del = 0
+                def process_date(date_val):
+                    try:
+                        logger.debug('按天分区去重', {"库": database, "表": table, "日期": date_val})
+                        dup_count, affected_rows = self._deduplicate_table(
+                            database, table, columns, dry_run, use_python_dedup,
+                            dedup_start_date=date_val, dedup_end_date=date_val,
+                            lock_table=False
+                        )
+                        return (dup_count, affected_rows, date_val, None)
+                    except Exception as e:
+                        logger.error('分区去重异常', {"库": database, "表": table, "日期": date_val, "异常": str(e), "func": sys._getframe().f_code.co_name})
+                        return (0, 0, date_val, str(e))
+                with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                    future_to_date = {executor.submit(process_date, date_val): date_val for date_val in all_dates}
+                    for future in concurrent.futures.as_completed(future_to_date):
+                        dup_count, affected_rows, date_val, err = future.result()
+                        if err:
+                            logger.warning('分区处理失败', {"库": database, "表": table, "日期": date_val, "异常": err, "func": sys._getframe().f_code.co_name})
+                        total_dup += dup_count
+                        total_del += affected_rows
+                return (total_dup, total_del)
+            # 获取去重列
             use_columns = columns or all_columns
             use_columns = [col for col in use_columns if col.lower() in all_columns_lower and col.lower() not in exclude_columns_lower]
             invalid_columns = set([col for col in (columns or []) if col.lower() not in all_columns_lower])
             if invalid_columns:
                 logger.warning('不存在的列', {"库": database, "表": table, "不存在以下列": invalid_columns, 'func': sys._getframe().f_code.co_name})
             if not use_columns:
-                logger.error('没有有效的去重列', {"库": database, "表": table})
+                logger.error('没有有效的去重列', {"库": database, "表": table, "func": sys._getframe().f_code.co_name})
                 return (0, 0)
-            # 统一用反引号包裹
-            column_list = ', '.join([f'`{col}`' for col in use_columns])
-            temp_table = self._make_safe_table_name(table, prefix=f"temp_", suffix=f"_dedup_{os.getpid()}_{threading.get_ident()}")
             pk = self.primary_key
-            # 主键判断也用小写
-            if pk.lower() not in all_columns_lower and pk != 'id':
-                logger.error('', {"不存在主键列": database, "表": table, "主键列不存在": pk})
-                return (0, 0)
-            # 找到实际主键名
             pk_real = next((c for c in all_columns if c.lower() == pk.lower()), pk)
-            # 构造where条件
-            where_time = ''
-            if use_time_filter:
-                where_time = f"WHERE `{time_col}` >= '{self._dedup_start_date}' AND `{time_col}` <= '{self._dedup_end_date}'"
+            # 判断是否需要加日期区间条件
+            where_sql = ''
+            if has_time_col and dedup_start_date and dedup_end_date:
+                where_sql = f"t.`{time_col}` >= '{dedup_start_date}' AND t.`{time_col}` <= '{dedup_end_date}'"
+            # 获取原始数据总量（只统计区间内数据）
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    count_where = f"WHERE `{time_col}` >= '{dedup_start_date}' AND `{time_col}` <= '{dedup_end_date}'" if has_time_col and dedup_start_date and dedup_end_date else ''
+                    count_sql = f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}` {count_where}"
+                    logger.debug('执行SQL', {'sql': count_sql})
+                    cursor.execute(count_sql)
+                    total_count_row = cursor.fetchone()
+                    total_count = total_count_row['cnt'] if total_count_row and 'cnt' in total_count_row else 0
+            logger.info('执行', {"库": database, "表": table, "开始处理数据量": total_count, 'func': sys._getframe().f_code.co_name, "数据日期": dedup_end_date})
+            column_list = ', '.join([f'`{col}`' for col in use_columns])
+
+            # 用Python查找重复
+            if use_python_dedup:
+                # 1. 拉取所有数据（生成器分批拉取）
+                select_cols = f'`{pk_real}`,' + ','.join([f'`{col}`' for col in use_columns])
+                select_where = f"WHERE `{time_col}` >= '{dedup_start_date}' AND `{time_col}` <= '{dedup_end_date}'" if has_time_col and dedup_start_date and dedup_end_date else ''
+                select_sql = f"SELECT {select_cols} FROM `{database}`.`{table}` {select_where}"
+                logger.debug('用Python查找重复，拉取数据SQL', {'sql': select_sql})
+                # 用生成器分批拉取
+                grouped = defaultdict(list)
+                for row in self._row_generator(database, table, select_cols, select_where, self.batch_size):
+                    key = tuple(row[col] for col in use_columns)
+                    grouped[key].append(row[pk_real])
+                # 2. 统计重复组和待删除id
+                dup_count = 0
+                del_ids = []
+                for ids in grouped.values():
+                    if len(ids) > 1:
+                        dup_count += 1
+                        del_ids.extend(ids[1:])  # 只保留第一个
+                affected_rows = 0
+                if not dry_run and del_ids:
+                    with self._get_connection() as conn:
+                        with conn.cursor() as cursor:
+                            for i in range(0, len(del_ids), self.batch_size):
+                                batch = del_ids[i:i+self.batch_size]
+                                del_ids_str = ','.join([str(i) for i in batch])
+                                delete_sql = f"DELETE FROM `{database}`.`{table}` WHERE `{pk_real}` IN ({del_ids_str})"
+                                logger.debug('用Python分批删除SQL', {'sql': delete_sql, 'ids': batch})
+                                cursor.execute(delete_sql)
+                                batch_deleted = cursor.rowcount
+                                affected_rows += batch_deleted
+                                conn.commit()
+                logger.info('去重完成', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "Python", "数据处理": self.duplicate_keep_mode, "数据日期": dedup_end_date})
+                return (dup_count, affected_rows)
+            # SQL方式查找重复
+            temp_table = self._make_temp_table_name(table)
+            drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
+            # 创建临时表时加where条件
+            create_temp_where = f"WHERE `{time_col}` >= '{dedup_start_date}' AND `{time_col}` <= '{dedup_end_date}'" if has_time_col and dedup_start_date and dedup_end_date else ''
             create_temp_sql = f"""
             CREATE TABLE `{database}`.`{temp_table}` AS
             SELECT MIN(`{pk_real}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
             FROM `{database}`.`{table}`
-            {where_time}
+            {create_temp_where}
             GROUP BY {column_list}
             HAVING COUNT(*) > 1
             """
-            drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
                     logger.debug('创建临时表SQL', {'sql': create_temp_sql})
                     cursor.execute(create_temp_sql)
-                    logger.debug('统计临时表重复组SQL', {'sql': f'SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`'})
                     cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`")
                     dup_count_row = cursor.fetchone()
                     dup_count = dup_count_row['cnt'] if dup_count_row and 'cnt' in dup_count_row else 0
                     if dup_count == 0:
-                        logger.info('没有重复数据', {"库": database, "表": table, "数据量": total_count, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns})
-                        logger.debug('删除临时表SQL', {'sql': drop_temp_sql})
+                        logger.info('没有重复数据', {"库": database, "表": table, "数据量": total_count, "数据日期": dedup_end_date})
                         cursor.execute(drop_temp_sql)
                         conn.commit()
                         return (0, 0)
                     affected_rows = 0
                     if not dry_run:
-                        # 分批删除，避免锁表
                         while True:
-                            if self.duplicate_keep_mode == 'remove_all':
-                                # 删除所有重复组的所有记录
-                                delete_dup_sql = f"""
-                                DELETE FROM `{database}`.`{table}`
-                                WHERE ({', '.join([f'`{col}`' for col in use_columns])}) IN (
-                                    SELECT {column_list} FROM `{database}`.`{temp_table}`
-                                ) {'AND' if use_time_filter else ''} {f'`{time_col}` >= \'{self._dedup_start_date}\' AND `{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
-                                LIMIT {self.batch_size}
-                                """
-                            else:
-                                # 修正：只删除重复组中不是min_id的行，唯一数据不动
-                                delete_dup_sql = f"""
-                                DELETE FROM `{database}`.`{table}` t
-                                WHERE EXISTS (
-                                    SELECT 1 FROM `{database}`.`{temp_table}` tmp
-                                    WHERE
-                                        {' AND '.join([f't.`{col}` <=> tmp.`{col}`' for col in use_columns])}
-                                        AND t.`{pk_real}` <> tmp.`min_id`
-                                )
-                                {'AND' if use_time_filter else ''} {f't.`{time_col}` >= \'{self._dedup_start_date}\' AND t.`{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
-                                LIMIT {self.batch_size}
-                                """
-                            logger.debug('执行删除重复数据SQL', {'sql': delete_dup_sql})
-                            cursor.execute(delete_dup_sql)
+                            where_clauses = []
+                            if self.duplicate_keep_mode == 'keep_one':
+                                where_clauses.append(f"t.`{pk_real}` <> tmp.`min_id`")
+                            if where_sql.strip():
+                                where_clauses.append(where_sql.strip())
+                            where_full = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
+                            find_dup_ids_sql = f"""
+                            SELECT t.`{pk_real}` as del_id
+                            FROM `{database}`.`{table}` t
+                            JOIN `{database}`.`{temp_table}` tmp
+                            ON {' AND '.join([f't.`{col}` <=> tmp.`{col}`' for col in use_columns])}
+                            {where_full}
+                            LIMIT {self.batch_size}
+                            """
+                            logger.debug('查找待删除重复id SQL', {'sql': find_dup_ids_sql})
+                            cursor.execute(find_dup_ids_sql)
+                            del_ids = [row['del_id'] for row in cursor.fetchall()]
+                            if not del_ids:
+                                break
+                            del_ids_str = ','.join([str(i) for i in del_ids])
+                            delete_sql = f"DELETE FROM `{database}`.`{table}` WHERE `{pk_real}` IN ({del_ids_str})"
+                            logger.debug('按id批量删除SQL', {'sql': delete_sql, 'ids': del_ids})
+                            cursor.execute(delete_sql)
                             batch_deleted = cursor.rowcount
                             affected_rows += batch_deleted
                             conn.commit()
+                            if batch_deleted == 0:
+                                logger.warning('检测到未能删除任何数据，强制跳出循环，防止假死', {"库": database, "表": table})
+                                break
                             if batch_deleted < self.batch_size:
                                 break
-                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组数": dup_count, "实际删除": affected_rows, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns, "去重模式": self.duplicate_keep_mode})
+                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组": dup_count, "实际删除": affected_rows, "去重方式": "SQL", "数据处理": self.duplicate_keep_mode, "数据日期": dedup_end_date})
                     else:
-                        logger.debug('dry_run模式，不执行删除', {"库": database, "表": table, "重复组数": dup_count, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None})
+                        logger.debug('dry_run模式，不执行删除', {"库": database, "表": table, "重复组": dup_count})
                         affected_rows = 0
-                    logger.debug('删除临时表SQL', {'sql': drop_temp_sql})
                     cursor.execute(drop_temp_sql)
                     conn.commit()
                     return (dup_count, affected_rows)
@@ -468,14 +607,17 @@ class MySQLDeduplicator:
                     logger.error('异常时清理临时表失败', {"库": database, "表": table, "异常": str(drop_e)})
             return (0, 0)
         finally:
-            self._release_table_lock(database, table)
+            if lock_table:
+                self._release_table_lock(database, table)
 
     def deduplicate_table(
             self,
             database: str,
             table: str,
             columns: Optional[List[str]] = None,
-            dry_run: bool = False
+            dry_run: bool = False,
+            reorder_id: bool = False,
+            use_python_dedup: bool = True
     ) -> Tuple[int, int]:
         """
         对指定表进行去重。
@@ -485,6 +627,8 @@ class MySQLDeduplicator:
             table (str): 表名。
             columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
             dry_run (bool): 是否为模拟运行（只统计不实际删除）。
+            reorder_id (bool): 去重后是否重排id。
+            use_python_dedup (bool): 是否用Python查找重复id。
         Returns:
             Tuple[int, int]: (重复组数, 实际删除行数)。
         """
@@ -495,9 +639,17 @@ class MySQLDeduplicator:
             if not self._check_table_exists(database, table):
                 logger.warning('表不存在', {"库": database, "表": table, "warning": "跳过"})
                 return (0, 0)
-            logger.info('单表开始', {"库": database, "表": table, "参数": {"指定去重列": columns, "模拟运行": dry_run, '排除列': self.exclude_columns}})
-            result = self._deduplicate_table(database, table, columns, dry_run)
+            logger.info('单表开始', {"库": database, "表": table, "参数": {"指定去重列": columns, "模拟运行": dry_run, '排除列': self.exclude_columns, 'use_python_dedup': use_python_dedup}})
+            result = self._deduplicate_table(database, table, columns, dry_run, use_python_dedup)
             logger.info('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result})
+            # 自动重排id列（仅当有实际删除时且reorder_id为True）
+            dup_count, affected_rows = result
+            if reorder_id and affected_rows > 0:
+                try:
+                    reorder_ok = self.reorder_id_column(database, table, id_column=self.primary_key, dry_run=dry_run)
+                    logger.info('自动重排id列完成', {"库": database, "表": table, "结果": reorder_ok})
+                except Exception as e:
+                    logger.error('自动重排id列异常', {"库": database, "表": table, "异常": str(e)})
             return result
         except Exception as e:
             logger.error('发生全局错误', {"库": database, "表": table, 'func': sys._getframe().f_code.co_name, "发生全局错误": str(e)})
@@ -509,7 +661,9 @@ class MySQLDeduplicator:
             tables: Optional[List[str]] = None,
             columns_map: Optional[Dict[str, List[str]]] = None,
             dry_run: bool = False,
-            parallel: bool = False
+            parallel: bool = False,
+            reorder_id: bool = False,
+            use_python_dedup: bool = True
     ) -> Dict[str, Tuple[int, int]]:
         """
         对指定数据库的所有表进行去重。
@@ -520,6 +674,8 @@ class MySQLDeduplicator:
             columns_map (Optional[Dict[str, List[str]]]): 各表使用的去重列 {表名: [列名]}。
             dry_run (bool): 是否为模拟运行。
             parallel (bool): 是否并行处理。
+            reorder_id (bool): 去重后是否重排id。
+            use_python_dedup (bool): 是否用Python查找重复id。
         Returns:
             Dict[str, Tuple[int, int]]: {表名: (重复组数, 实际删除行数)}。
         """
@@ -548,7 +704,7 @@ class MySQLDeduplicator:
                         logger.debug('提交表去重任务', {'库': database, '表': table, 'columns': columns})
                         futures[executor.submit(
                             self.deduplicate_table,
-                            database, table, columns, dry_run
+                            database, table, columns, dry_run, reorder_id, use_python_dedup
                         )] = table
                     for future in concurrent.futures.as_completed(futures):
                         table = futures[future]
@@ -564,12 +720,12 @@ class MySQLDeduplicator:
                 for table in target_tables:
                     columns = columns_map.get(table) if columns_map else None
                     dup_count, affected_rows = self.deduplicate_table(
-                        database, table, columns, dry_run
+                        database, table, columns, dry_run, reorder_id, use_python_dedup
                     )
                     results[table] = (dup_count, affected_rows)
             total_dup = sum(r[0] for r in results.values())
             total_del = sum(r[1] for r in results.values())
-            logger.info('单库完成', {"库": database, "重复组数": total_dup, "总删除行数": total_del, "详细结果": results})
+            logger.info('单库完成', {"库": database, "重复组": total_dup, "总删除行": total_del, "详细结果": results})
             return results
         except Exception as e:
             logger.error('发生全局错误', {"库": database, 'func': sys._getframe().f_code.co_name, "error": str(e), 'traceback': repr(e)})
@@ -581,7 +737,9 @@ class MySQLDeduplicator:
             tables_map: Optional[Dict[str, List[str]]] = None,
             columns_map: Optional[Dict[str, Dict[str, List[str]]]] = None,
             dry_run: bool = False,
-            parallel: bool = False
+            parallel: bool = False,
+            reorder_id: bool = False,
+            use_python_dedup: bool = True
     ) -> Dict[str, Dict[str, Tuple[int, int]]]:
         """
         对所有数据库进行去重。
@@ -592,6 +750,8 @@ class MySQLDeduplicator:
             columns_map (Optional[Dict[str, Dict[str, List[str]]]]): 指定每个表去重时使用的列，格式为 {数据库名: {表名: [列名, ...]}}。如果为 None，则使用所有列。
             dry_run (bool): 是否为模拟运行模式。为 True 时只统计重复行数，不实际删除。
             parallel (bool): 是否并行处理多个数据库。为 True 时使用线程池并发处理。
+            reorder_id (bool): 去重后是否重排id。
+            use_python_dedup (bool): 是否用Python查找重复id。
         Returns:
             Dict[str, Dict[str, Tuple[int, int]]]: 嵌套字典，格式为 {数据库名: {表名: (重复组数, 实际删除行数)}}。
         """
@@ -603,7 +763,7 @@ class MySQLDeduplicator:
             if not target_dbs:
                 logger.warning('没有可处理的数据库')
                 return all_results
-            logger.info('全局开始', {"数据库数量": len(target_dbs), "数据库列表": target_dbs, "参数": {"模拟运行": dry_run, "并行处理": parallel, '排除列': self.exclude_columns}})
+            logger.info('全局开始', {"数据库数量": len(target_dbs), "数据库列表": target_dbs, "参数": {"模拟运行": dry_run, "并行处理": parallel, '排除列': self.exclude_columns, 'use_python_dedup': use_python_dedup}})
             if parallel and self.max_workers > 1:
                 # 使用线程池并行处理多个数据库
                 with concurrent.futures.ThreadPoolExecutor(
@@ -615,7 +775,7 @@ class MySQLDeduplicator:
                         db_columns_map = columns_map.get(db) if columns_map else None
                         futures[executor.submit(
                             self.deduplicate_database,
-                            db, tables, db_columns_map, dry_run, False
+                            db, tables, db_columns_map, dry_run, False, reorder_id, use_python_dedup
                         )] = db
                     for future in concurrent.futures.as_completed(futures):
                         db = futures[future]
@@ -631,7 +791,7 @@ class MySQLDeduplicator:
                     tables = tables_map.get(db) if tables_map else None
                     db_columns_map = columns_map.get(db) if columns_map else None
                     db_results = self.deduplicate_database(
-                        db, tables, db_columns_map, dry_run, parallel
+                        db, tables, db_columns_map, dry_run, parallel, reorder_id, use_python_dedup
                     )
                     all_results[db] = db_results
             total_dup = sum(
@@ -642,7 +802,7 @@ class MySQLDeduplicator:
                 r[1] for db in all_results.values()
                 for r in db.values()
             )
-            logger.info('全局完成', {"总重复组数": total_dup, "总删除行数": total_del, "详细结果": dict(all_results)})
+            logger.info('全局完成', {"总重复组": total_dup, "总删除行": total_del, "详细结果": dict(all_results)})
             return all_results
         except Exception as e:
             logger.error('异常', {"error": str(e), 'traceback': repr(e)})
@@ -772,22 +932,12 @@ class MySQLDeduplicator:
             if not self._check_table_exists(database, table):
                 logger.warning('表不存在，跳过id重排', {"库": database, "表": table})
                 return False
-            # 检查id列是否存在
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    cursor.execute("""
-                        SELECT COLUMN_NAME, COLUMN_KEY
-                        FROM INFORMATION_SCHEMA.COLUMNS
-                        WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s
-                    """, (database, table))
-                    columns_info = cursor.fetchall()
-                    columns = [row['COLUMN_NAME'] for row in columns_info]
-                    id_is_pk = any(row['COLUMN_NAME'].lower() == id_column.lower() and row['COLUMN_KEY'] in ('PRI', 'UNI') for row in columns_info)
+            # 检查id列、主键信息（用_get_table_info）
+            columns, pk_cols, id_is_pk = self._get_table_info(database, table, id_column)
             if id_column not in columns:
                 logger.warning('表无id列，跳过id重排', {"库": database, "表": table})
                 return False
             # 检查主键是否为单列id
-            pk_cols = [row['COLUMN_NAME'] for row in columns_info if row['COLUMN_KEY'] == 'PRI']
             if len(pk_cols) != 1 or pk_cols[0].lower() != id_column.lower():
                 logger.warning('主键不是单列id，跳过id重排', {"库": database, "表": table, "主键列": pk_cols})
                 return False
@@ -806,13 +956,13 @@ class MySQLDeduplicator:
                 with conn.cursor() as cursor:
                     cursor.execute(f"SHOW CREATE TABLE {table_quoted}")
                     create_table_sql = cursor.fetchone()['Create Table']
-            logger.info('开始id重排', {"库": database, "表": table, "重排列": id_column, "dry_run": dry_run, "DDL警告": "MySQL DDL操作不可回滚，建议提前备份！"})
+            logger.info('开始id重排', {"库": database, "表": table, "重排列": id_column, "试运行": dry_run, "DDL警告": "MySQL DDL操作不可回滚，建议提前备份！"})
             if dry_run:
                 logger.info('dry_run模式，打印原表结构', {"库": database, "表": table, "建表语句": create_table_sql})
                 return True
-            temp_table = self._make_safe_table_name(table, prefix=f"temp_", suffix=f"_reorderid_{os.getpid()}_{threading.get_ident()}")
+            temp_table = self._make_temp_table_name(table)
             temp_table_quoted = f"`{database}`.`{temp_table}`"
-            backup_table = self._make_safe_table_name(table, prefix="backup_", suffix=f"_{int(time.time())}_{uuid.uuid4().hex[:8]}")
+            backup_table = self._make_backup_table_name(table)
             backup_table_quoted = f"`{database}`.`{backup_table}`"
             try:
                 with self._get_connection() as conn:
@@ -927,23 +1077,71 @@ class MySQLDeduplicator:
             return (prefix + suffix)[:max_length]
         return f"{prefix}{base[:remain]}{suffix}"[:max_length]
 
+    def _get_table_info(self, database: str, table: str, id_column: str = None):
+        """
+        获取表的所有列名、主键列名列表、指定id列是否为主键。
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            id_column (str): id列名，默认使用self.primary_key。
+        Returns:
+            Tuple[List[str], List[str], bool]: (所有列名, 主键列名, id列是否为主键)
+        """
+        id_column = id_column or self.primary_key
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute("""
+                    SELECT COLUMN_NAME, COLUMN_KEY
+                    FROM INFORMATION_SCHEMA.COLUMNS
+                    WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s
+                """, (database, table))
+                columns_info = cursor.fetchall()
+                columns = [row['COLUMN_NAME'] for row in columns_info]
+                pk_cols = [row['COLUMN_NAME'] for row in columns_info if row['COLUMN_KEY'] == 'PRI']
+                id_is_pk = any(row['COLUMN_NAME'].lower() == id_column.lower() and row['COLUMN_KEY'] in ('PRI', 'UNI') for row in columns_info)
+        return columns, pk_cols, id_is_pk
+
+    def _make_temp_table_name(self, base: str) -> str:
+        """
+        生成临时表名，带有 temp_ 前缀和 _dedup_ 进程线程后缀。
+        """
+        suffix = f"_dedup_{os.getpid()}_{threading.get_ident()}"
+        return self._make_safe_table_name(base, prefix="temp_", suffix=suffix)
+
+    def _make_backup_table_name(self, base: str) -> str:
+        """
+        生成备份表名，带有 backup_ 前缀和时间戳+uuid后缀。
+        """
+        suffix = f"_{int(time.time())}_{uuid.uuid4().hex[:8]}"
+        return self._make_safe_table_name(base, prefix="backup_", suffix=suffix)
+
 
 def main():
     deduplicator = MySQLDeduplicator(
         username='root',
         password='pwd',
         host='localhost',
-        port=3306
+        port=3306,
+        # date_range=['2025-05-27', '2025-05-28'],
+        exclude_tables={'推广数据2': [
+            # '地域报表_城市_2025_04', 
+            # '地域报表_城市_2025_05', 
+            # '地域报表_城市_2025_06', 
+            '地域报表_城市_2025_04_copy1', 
+            '地域报表_城市_2025_05_copy1', 
+            '地域报表_城市_2025_06_copy1', 
+            '主体报表_2025_copy1'
+            ]}
     )
 
     # 全库去重(单线程)
-    deduplicator.deduplicate_all(dry_run=False, parallel=True)
+    deduplicator.deduplicate_all(dry_run=True, parallel=True, reorder_id=True)
 
     # # 指定数据库去重(多线程)
-    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=False)
+    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=False, reorder_id=True)
 
     # # 指定表去重(使用特定列)
-    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'], dry_run=False)
+    # deduplicator.deduplicate_table('my_db', 'my_table', columns=["name", "date"], dry_run=False, reorder_id=False)
 
     # # 重排id列
     # deduplicator.reorder_id_column('my_db', 'my_table', 'id', dry_run=False, auto_drop_backup=True)
@@ -952,5 +1150,5 @@ def main():
     deduplicator.close()
 
 if __name__ == '__main__':
-    main()
+    # main()
     pass

mdbq/mysql/uploader.py

@@ -428,6 +428,7 @@ class MySQLUploader:
                 if idx_col in set_typ:
                     safe_idx_col = self._validate_identifier(idx_col)
                     index_defs.append(f"INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)")
+        index_defs = list(set(index_defs))
         index_sql = (',' + ','.join(index_defs)) if index_defs else ''
         sql = f"""
         CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
@@ -593,6 +594,34 @@ class MySQLUploader:
             logger.error('无法获取表列信息', {'库': db_name, '表': table_name, '错误': str(e)})
             raise
 
+    def _ensure_index(self, db_name: str, table_name: str, column: str):
+        """
+        确保某列有索引，如果没有则创建。
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        column = self._validate_identifier(column)
+        # 检查索引是否已存在
+        sql_check = '''
+            SELECT COUNT(1) FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s
+        '''
+        sql_create = f'ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{column}` (`{column}`)'
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql_check, (db_name, table_name, column))
+                    exists = cursor.fetchone()
+                    if exists and list(exists.values())[0] > 0:
+                        logger.debug('索引已存在', {'库': db_name, '表': table_name, '列': column})
+                        return
+                    cursor.execute(sql_create)
+                conn.commit()
+                logger.info('已为列创建索引', {'库': db_name, '表': table_name, '列': column})
+        except Exception as e:
+            logger.error('创建索引失败', {'库': db_name, '表': table_name, '列': column, '错误': str(e)})
+            raise
+
     def _upload_to_table(
             self,
             db_name: str,
@@ -646,6 +675,13 @@ class MySQLUploader:
                 })
                 raise ValueError(f"列不存在: `{col}` -> `{db_name}`.`{table_name}`")
 
+        # 确保分表参考字段为索引
+        if date_column and date_column in table_columns:
+            try:
+                self._ensure_index(db_name, table_name, date_column)
+            except Exception as e:
+                logger.warning('分表参考字段索引创建失败', {'库': db_name, '表': table_name, '列': date_column, '错误': str(e)})
+
         # 插入数据
         self._insert_data(
             db_name, table_name, data, set_typ,
@@ -868,7 +904,7 @@ class MySQLUploader:
         :param duplicate_columns: 用于检查重复的列，可选
         :param allow_null: 是否允许空值，默认为False
         :param partition_by: 分表方式('year'、'month'、'None')，可选
-        :param partition_date_column: 用于分表的日期列名，默认为'日期'
+        :param partition_date_column: 用于分表的日期列名，默认为'日期', 默认会添加为索引
         :param auto_create: 表不存在时是否自动创建，默认为True
         :param indexes: 需要创建索引的列列表，可选
         :param update_on_duplicate: 遇到重复数据时是否更新旧数据，默认为False
@@ -977,6 +1013,12 @@ class MySQLUploader:
                             allow_null, auto_create, partition_date_column,
                             indexes, batch_id, update_on_duplicate, transaction_mode
                         )
+                        # 确保分表参考字段为索引
+                        if partition_date_column in filtered_set_typ:
+                            try:
+                                self._ensure_index(db_name, part_table, partition_date_column)
+                            except Exception as e:
+                                logger.warning('分表参考字段索引创建失败', {'库': db_name, '表': part_table, '列': partition_date_column, '错误': str(e)})
                     except Exception as e:
                         logger.error('分表上传异常', {
                             '库': db_name,
@@ -995,6 +1037,12 @@ class MySQLUploader:
                     allow_null, auto_create, partition_date_column,
                     indexes, batch_id, update_on_duplicate, transaction_mode
                 )
+                # 确保分表参考字段为索引
+                if partition_date_column in filtered_set_typ:
+                    try:
+                        self._ensure_index(db_name, table_name, partition_date_column)
+                    except Exception as e:
+                        logger.warning('分表参考字段索引创建失败', {'库': db_name, '表': table_name, '列': partition_date_column, '错误': str(e)})
 
             success_flag = True
 

{mdbq-3.11.8.dist-info → mdbq-3.11.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.11.8
+Version: 3.11.10
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.11.8.dist-info → mdbq-3.11.10.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=JqV56ilza72jpkf_fztVtAdeSmcdPr0BmGGo9FFjGrA,18
+mdbq/__version__.py,sha256=L9HK2W1LgO8Zc5gpJgI1uJ5J0VRcUyMXHr1ZT-FeNOM,19
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=HuxLBCXjm6fZrxYE0rdpUCz359WGeqOX0vvg9jTuRY4,24126
+mdbq/log/mylogger.py,sha256=Crw6LwVo3I3IUbzIETu8f46Quza3CTCh-qYf4edbBPo,24139
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=Znmjn4sI1Mj2koSPTDojFwg_1MTgk3GZTFZyhSRwn7s,46746
+mdbq/mysql/deduplicator.py,sha256=w8etA5dAsY7g58bWU3SQt7n_OWnS9Y2TVh0D7m0MK9E,57961
 mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
-mdbq/mysql/uploader.py,sha256=LxPlAfSNhQbLu-or4wxa-vLjCw5_PIN3ZVoksWUJazQ,61701
+mdbq/mysql/uploader.py,sha256=8Px_W2bYOr1wQgMXMK0DggNiuE6a6Ul4BlJake8LSo8,64469
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -24,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
-mdbq-3.11.8.dist-info/METADATA,sha256=EJtaHsIzWmcB9hTRg1NZeDd55Zez0lu6FPD_ZQB9nMw,364
-mdbq-3.11.8.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.11.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.11.8.dist-info/RECORD,,
+mdbq-3.11.10.dist-info/METADATA,sha256=dVhkC84iq1GWtV6onfsLj18CwfGnIo1bXXDa-TXUU1E,365
+mdbq-3.11.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.11.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.11.10.dist-info/RECORD,,