mdbq 3.11.5__py3-none-any.whl → 3.11.7__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.11.5'
+VERSION = '3.11.7'

mdbq/mysql/deduplicator.py

@@ -80,7 +80,8 @@ class MySQLDeduplicator:
             date_column: str = '日期',
             exclude_columns: Optional[List[str]] = None,
             exclude_databases: Optional[List[str]] = None,
-            exclude_tables: Optional[Dict[str, List[str]]] = None
+            exclude_tables: Optional[Dict[str, List[str]]] = None,
+            duplicate_keep_mode: str = 'keep_one'  # 新增参数
     ) -> None:
         """
         初始化去重处理器
@@ -90,6 +91,7 @@ class MySQLDeduplicator:
         :param exclude_columns: 去重时排除的列名列表，默认为['id', '更新时间']
         :param exclude_databases: 排除的数据库名列表
         :param exclude_tables: 排除的表名字典 {数据库名: [表名, ...]}
+        :param duplicate_keep_mode: 'keep_one'（默认，重复组保留一条），'remove_all'（全部删除重复组）
         """
         # 连接池状态标志
         self._closed = False
@@ -173,6 +175,8 @@ class MySQLDeduplicator:
         self.exclude_databases = set([db.lower() for db in exclude_databases]) if exclude_databases else set()
         self.exclude_tables = {k.lower(): set([t.lower() for t in v]) for k, v in (exclude_tables or {}).items()}
 
+        self.duplicate_keep_mode = duplicate_keep_mode if duplicate_keep_mode in ('keep_one', 'remove_all') else 'keep_one'
+
     def _get_connection(self) -> pymysql.connections.Connection:
         """
         从连接池获取一个数据库连接。
@@ -329,7 +333,8 @@ class MySQLDeduplicator:
             database: str,
             table: str,
             columns: Optional[List[str]] = None,
-            dry_run: bool = False
+            dry_run: bool = False,
+            reset_id: bool = False
     ) -> Tuple[int, int]:
         """
         执行单表去重。
@@ -339,6 +344,7 @@ class MySQLDeduplicator:
             table (str): 表名。
             columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
             dry_run (bool): 是否为模拟运行（只统计不实际删除）。
+            reset_id (bool): 是否在去重后重排id。
         Returns:
             Tuple[int, int]: (重复组数, 实际删除行数)。
         """
@@ -414,14 +420,28 @@ class MySQLDeduplicator:
                     if not dry_run:
                         # 分批删除，避免锁表
                         while True:
-                            delete_dup_sql = f"""
-                            DELETE FROM `{database}`.`{table}`
-                            WHERE `{pk_real}` NOT IN (
-                                SELECT `min_id` FROM `{database}`.`{temp_table}`
-                            ) {'AND' if use_time_filter else ''} {f'`{time_col}` >= \'{self._dedup_start_date}\' AND `{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
-                            AND ({' AND '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
-                            LIMIT {self.batch_size}
-                            """
+                            if self.duplicate_keep_mode == 'remove_all':
+                                # 删除所有重复组的所有记录
+                                delete_dup_sql = f"""
+                                DELETE FROM `{database}`.`{table}`
+                                WHERE ({', '.join([f'`{col}`' for col in use_columns])}) IN (
+                                    SELECT {column_list} FROM `{database}`.`{temp_table}`
+                                ) {'AND' if use_time_filter else ''} {f'`{time_col}` >= \'{self._dedup_start_date}\' AND `{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
+                                LIMIT {self.batch_size}
+                                """
+                            else:
+                                # 修正：只删除重复组中不是min_id的行，唯一数据不动
+                                delete_dup_sql = f"""
+                                DELETE FROM `{database}`.`{table}` t
+                                WHERE EXISTS (
+                                    SELECT 1 FROM `{database}`.`{temp_table}` tmp
+                                    WHERE
+                                        {' AND '.join([f't.`{col}` <=> tmp.`{col}`' for col in use_columns])}
+                                        AND t.`{pk_real}` <> tmp.`min_id`
+                                )
+                                {'AND' if use_time_filter else ''} {f't.`{time_col}` >= \'{self._dedup_start_date}\' AND t.`{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
+                                LIMIT {self.batch_size}
+                                """
                             logger.debug('执行删除重复数据SQL', {'sql': delete_dup_sql})
                             cursor.execute(delete_dup_sql)
                             batch_deleted = cursor.rowcount
@@ -429,7 +449,10 @@ class MySQLDeduplicator:
                             conn.commit()
                             if batch_deleted < self.batch_size:
                                 break
-                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组数": dup_count, "实际删除": affected_rows, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns})
+                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组数": dup_count, "实际删除": affected_rows, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns, "去重模式": self.duplicate_keep_mode})
+                        # 新增：去重后重排id
+                        if reset_id and affected_rows > 0:
+                            self._reset_id_column(database, table)
                     else:
                         logger.debug('dry_run模式，不执行删除', {"库": database, "表": table, "重复组数": dup_count, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None})
                         affected_rows = 0
@@ -458,7 +481,8 @@ class MySQLDeduplicator:
             database: str,
             table: str,
             columns: Optional[List[str]] = None,
-            dry_run: bool = False
+            dry_run: bool = False,
+            reset_id: bool = False
     ) -> Tuple[int, int]:
         """
         对指定表进行去重。
@@ -468,6 +492,7 @@ class MySQLDeduplicator:
             table (str): 表名。
             columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
             dry_run (bool): 是否为模拟运行（只统计不实际删除）。
+            reset_id (bool): 是否在去重后重排id。
         Returns:
             Tuple[int, int]: (重复组数, 实际删除行数)。
         """
@@ -479,7 +504,7 @@ class MySQLDeduplicator:
                 logger.warning('表不存在', {"库": database, "表": table, "warning": "跳过"})
                 return (0, 0)
             logger.info('单表开始', {"库": database, "表": table, "参数": {"指定去重列": columns, "模拟运行": dry_run, '排除列': self.exclude_columns}})
-            result = self._deduplicate_table(database, table, columns, dry_run)
+            result = self._deduplicate_table(database, table, columns, dry_run, reset_id)
             logger.info('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result})
             return result
         except Exception as e:
@@ -492,7 +517,8 @@ class MySQLDeduplicator:
             tables: Optional[List[str]] = None,
             columns_map: Optional[Dict[str, List[str]]] = None,
             dry_run: bool = False,
-            parallel: bool = False
+            parallel: bool = False,
+            reset_id: bool = False
     ) -> Dict[str, Tuple[int, int]]:
         """
         对指定数据库的所有表进行去重。
@@ -503,6 +529,7 @@ class MySQLDeduplicator:
             columns_map (Optional[Dict[str, List[str]]]): 各表使用的去重列 {表名: [列名]}。
             dry_run (bool): 是否为模拟运行。
             parallel (bool): 是否并行处理。
+            reset_id (bool): 是否在去重后重排id。
         Returns:
             Dict[str, Tuple[int, int]]: {表名: (重复组数, 实际删除行数)}。
         """
@@ -531,7 +558,7 @@ class MySQLDeduplicator:
                         logger.debug('提交表去重任务', {'库': database, '表': table, 'columns': columns})
                         futures[executor.submit(
                             self.deduplicate_table,
-                            database, table, columns, dry_run
+                            database, table, columns, dry_run, reset_id
                         )] = table
                     for future in concurrent.futures.as_completed(futures):
                         table = futures[future]
@@ -547,7 +574,7 @@ class MySQLDeduplicator:
                 for table in target_tables:
                     columns = columns_map.get(table) if columns_map else None
                     dup_count, affected_rows = self.deduplicate_table(
-                        database, table, columns, dry_run
+                        database, table, columns, dry_run, reset_id
                     )
                     results[table] = (dup_count, affected_rows)
             total_dup = sum(r[0] for r in results.values())
@@ -564,7 +591,8 @@ class MySQLDeduplicator:
             tables_map: Optional[Dict[str, List[str]]] = None,
             columns_map: Optional[Dict[str, Dict[str, List[str]]]] = None,
             dry_run: bool = False,
-            parallel: bool = False
+            parallel: bool = False,
+            reset_id: bool = False
     ) -> Dict[str, Dict[str, Tuple[int, int]]]:
         """
         对所有数据库进行去重。
@@ -575,6 +603,7 @@ class MySQLDeduplicator:
             columns_map (Optional[Dict[str, Dict[str, List[str]]]]): 指定每个表去重时使用的列，格式为 {数据库名: {表名: [列名, ...]}}。如果为 None，则使用所有列。
             dry_run (bool): 是否为模拟运行模式。为 True 时只统计重复行数，不实际删除。
             parallel (bool): 是否并行处理多个数据库。为 True 时使用线程池并发处理。
+            reset_id (bool): 是否在去重后重排id。
         Returns:
             Dict[str, Dict[str, Tuple[int, int]]]: 嵌套字典，格式为 {数据库名: {表名: (重复组数, 实际删除行数)}}。
         """
@@ -598,7 +627,7 @@ class MySQLDeduplicator:
                         db_columns_map = columns_map.get(db) if columns_map else None
                         futures[executor.submit(
                             self.deduplicate_database,
-                            db, tables, db_columns_map, dry_run, False
+                            db, tables, db_columns_map, dry_run, False, reset_id
                         )] = db
                     for future in concurrent.futures.as_completed(futures):
                         db = futures[future]
@@ -614,7 +643,7 @@ class MySQLDeduplicator:
                     tables = tables_map.get(db) if tables_map else None
                     db_columns_map = columns_map.get(db) if columns_map else None
                     db_results = self.deduplicate_database(
-                        db, tables, db_columns_map, dry_run, parallel
+                        db, tables, db_columns_map, dry_run, parallel, reset_id
                     )
                     all_results[db] = db_results
             total_dup = sum(
@@ -709,6 +738,146 @@ class MySQLDeduplicator:
         """
         self.close()
 
+    def _has_foreign_key_dependency(self, database: str, table: str, pk: str) -> bool:
+        """检测id列是否被其他表外键引用。"""
+        fk_check_sql = '''
+        SELECT TABLE_NAME, COLUMN_NAME, CONSTRAINT_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
+        FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
+        WHERE REFERENCED_TABLE_SCHEMA = %s AND REFERENCED_TABLE_NAME = %s AND REFERENCED_COLUMN_NAME = %s
+        '''
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(f"USE `{database}`")
+                cursor.execute(fk_check_sql, (database, table, pk))
+                fk_rows = cursor.fetchall()
+                return bool(fk_rows)
+
+    def _get_table_create_sql_and_pk(self, database: str, table: str) -> tuple:
+        """获取表的CREATE语句和主键字段列表。"""
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(f"USE `{database}`")
+                cursor.execute(f"SHOW CREATE TABLE `{database}`.`{table}`")
+                create_sql = cursor.fetchone()['Create Table']
+                cursor.execute(f"SHOW KEYS FROM `{database}`.`{table}` WHERE Key_name = 'PRIMARY'")
+                pk_rows = cursor.fetchall()
+                pk_columns = [row['Column_name'] for row in pk_rows]
+        return create_sql, pk_columns
+
+    def _make_temp_table_sql(self, create_sql: str, table: str, temp_table: str, pk: str, pk_columns: list) -> str:
+        """生成临时表的CREATE语句，仅替换id字段类型。"""
+        def replace_id_type(sql):
+            lines = sql.split('\n')
+            new_lines = []
+            for line in lines:
+                if re.match(rf'\s*`{pk}` ', line):
+                    if pk_columns == [pk]:
+                        line = re.sub(r'`' + pk + r'`\s+[^,]*', f'`{pk}` INT NOT NULL AUTO_INCREMENT', line)
+                    else:
+                        line = re.sub(r'`' + pk + r'`\s+[^,]*', f'`{pk}` INT NOT NULL', line)
+                new_lines.append(line)
+            return '\n'.join(new_lines)
+        create_sql_temp = re.sub(
+            rf'CREATE TABLE `{table}`',
+            f'CREATE TABLE `{temp_table}`',
+            create_sql,
+            count=1
+        )
+        create_sql_temp = replace_id_type(create_sql_temp)
+        create_sql_temp = re.sub(r'AUTO_INCREMENT=\d+', '', create_sql_temp)
+        return create_sql_temp
+
+    def _create_and_fill_temp_table(self, database: str, table: str, temp_table: str, pk: str) -> list:
+        """创建临时表并插入重排id数据，返回所有字段名。"""
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(f"USE `{database}`")
+                cursor.execute(f"SHOW COLUMNS FROM `{database}`.`{table}`")
+                columns = [row['Field'] for row in cursor.fetchall()]
+                columns_wo_id = [col for col in columns if col != pk]
+                col_list = ', '.join([f'`{col}`' for col in columns_wo_id])
+                insert_sql = f"INSERT INTO `{database}`.`{temp_table}` ({col_list}, `{pk}`) SELECT {col_list}, (@rownum:=@rownum+1) as `{pk}` FROM `{database}`.`{table}` JOIN (SELECT @rownum:=0) r ORDER BY `{pk}` ASC"
+                cursor.execute(insert_sql)
+                return columns
+
+    def _swap_tables_with_backup(self, database: str, table: str, temp_table: str, bak_table: str):
+        """原表重命名为备份，临时表变原表名。"""
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(f"USE `{database}`")
+                cursor.execute(f"RENAME TABLE `{database}`.`{table}` TO `{database}`.`{bak_table}`")
+                cursor.execute(f"RENAME TABLE `{database}`.`{temp_table}` TO `{database}`.`{table}`")
+                conn.commit()
+
+    def _check_and_cleanup_backup(self, database: str, table: str, bak_table: str) -> bool:
+        """校验新表和备份表数据量一致，安全删除备份表。"""
+        with self._get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute(f"USE `{database}`")
+                cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}`")
+                new_cnt = cursor.fetchone()['cnt']
+                cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{bak_table}`")
+                old_cnt = cursor.fetchone()['cnt']
+                if new_cnt == old_cnt:
+                    cursor.execute(f"DROP TABLE `{database}`.`{bak_table}`")
+                    conn.commit()
+                    return True
+                else:
+                    logger.error('id重排后数据量不一致，未删除备份表', {'库': database, '表': table, '新表行数': new_cnt, '备份表行数': old_cnt})
+                    return False
+
+    def _rollback_table_swap(self, database: str, table: str, bak_table: str):
+        """回滚：如bak表存在且原表不存在，则恢复原表名。"""
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(f"USE `{database}`")
+                    cursor.execute(f"SHOW TABLES LIKE '{bak_table}'")
+                    if cursor.fetchone():
+                        cursor.execute(f"SHOW TABLES LIKE '{table}'")
+                        if not cursor.fetchone():
+                            cursor.execute(f"RENAME TABLE `{database}`.`{bak_table}` TO `{database}`.`{table}`")
+                            conn.commit()
+                            logger.info('回滚成功，已恢复原表', {'库': database, '表': table})
+        except Exception as e2:
+            logger.error('回滚失败', {'库': database, '表': table, '异常': str(e2)})
+
+    def _reset_id_column(self, database: str, table: str) -> bool:
+        pk = self.primary_key
+        temp_table = f"temp_{table}_resetid_{os.getpid()}_{threading.get_ident()}"
+        temp_table = re.sub(r'[^a-zA-Z0-9_]', '_', temp_table)[:60]
+        bak_table = f"{table}_bak_{int(time.time())}"
+        try:
+            # 1. 检查外键依赖
+            if self._has_foreign_key_dependency(database, table, pk):
+                logger.warning('存在外键依赖，拒绝重排id', {'库': database, '表': table})
+                return False
+            # 2. 获取表结构和主键
+            create_sql, pk_columns = self._get_table_create_sql_and_pk(database, table)
+            # 3. 生成临时表DDL
+            create_sql_temp = self._make_temp_table_sql(create_sql, table, temp_table, pk, pk_columns)
+            # 4. 创建临时表
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(f"USE `{database}`")
+                    cursor.execute(f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`")
+                    cursor.execute(create_sql_temp)
+                    conn.commit()
+            # 5. 填充临时表
+            self._create_and_fill_temp_table(database, table, temp_table, pk)
+            # 6. 表交换
+            self._swap_tables_with_backup(database, table, temp_table, bak_table)
+            # 7. 校验和清理
+            if self._check_and_cleanup_backup(database, table, bak_table):
+                logger.info('id重排完成并安全删除备份表，主键信息已保留', {'库': database, '表': table})
+                return True
+            else:
+                return False
+        except Exception as e:
+            logger.error('id重排失败，尝试回滚', {'库': database, '表': table, '异常': str(e)})
+            self._rollback_table_swap(database, table, bak_table)
+            return False
+
 
 def main():
     deduplicator = MySQLDeduplicator(
@@ -719,14 +888,13 @@ def main():
     )
 
     # 全库去重(单线程)
-    deduplicator.deduplicate_all(dry_run=False, parallel=False)
+    deduplicator.deduplicate_all(dry_run=False, parallel=False, reset_id=False)
 
     # # 指定数据库去重(多线程)
-    # logger.info('调用deduplicate_database')
-    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=True)
+    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=True, reset_id=False)
 
     # # 指定表去重(使用特定列)
-    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'], dry_run=False)
+    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'], dry_run=False, reset_id=False)
 
     # 关闭连接
     deduplicator.close()

{mdbq-3.11.5.dist-info → mdbq-3.11.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.11.5
+Version: 3.11.7
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.11.5.dist-info → mdbq-3.11.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=Ne7M_nVkNZ4zJcEt-_2BhpBw01sgUpRFBru080buHJk,18
+mdbq/__version__.py,sha256=KXKzNBZD4M6L-jW29owhKjBycLSiUdGeTf_uNAYvyGI,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -8,7 +8,7 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=HuxLBCXjm6fZrxYE0rdpUCz359WGeqOX0vvg9jTuRY4,24126
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=bIV010UkFfSUONY6-756x3tDVO4k6q3pqxoY3Z2xT-k,32990
+mdbq/mysql/deduplicator.py,sha256=dq40YBLVd5ho28pYzVfwm5pA90YA3iN6l9xX4k0Ynds,42808
 mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
 mdbq/mysql/uploader.py,sha256=LxPlAfSNhQbLu-or4wxa-vLjCw5_PIN3ZVoksWUJazQ,61701
@@ -24,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
-mdbq-3.11.5.dist-info/METADATA,sha256=OxaQPCqcPoi2t8FFpw2eCKnde3B3ci-665xalC4GZT0,364
-mdbq-3.11.5.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.11.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.11.5.dist-info/RECORD,,
+mdbq-3.11.7.dist-info/METADATA,sha256=j_0kmOn4tTbk8TY8LqbEZ2OWmJz0-70sUQNwP_N0VCc,364
+mdbq-3.11.7.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.11.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.11.7.dist-info/RECORD,,