mdbq 3.11.6__py3-none-any.whl → 3.11.7__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.11.6'
+VERSION = '3.11.7'

mdbq/mysql/deduplicator.py

@@ -80,7 +80,8 @@ class MySQLDeduplicator:
             date_column: str = '日期',
             exclude_columns: Optional[List[str]] = None,
             exclude_databases: Optional[List[str]] = None,
-            exclude_tables: Optional[Dict[str, List[str]]] = None
+            exclude_tables: Optional[Dict[str, List[str]]] = None,
+            duplicate_keep_mode: str = 'keep_one'  # 新增参数
     ) -> None:
         """
         初始化去重处理器
@@ -90,6 +91,7 @@ class MySQLDeduplicator:
         :param exclude_columns: 去重时排除的列名列表，默认为['id', '更新时间']
         :param exclude_databases: 排除的数据库名列表
         :param exclude_tables: 排除的表名字典 {数据库名: [表名, ...]}
+        :param duplicate_keep_mode: 'keep_one'（默认，重复组保留一条），'remove_all'（全部删除重复组）
         """
         # 连接池状态标志
         self._closed = False
@@ -173,6 +175,8 @@ class MySQLDeduplicator:
         self.exclude_databases = set([db.lower() for db in exclude_databases]) if exclude_databases else set()
         self.exclude_tables = {k.lower(): set([t.lower() for t in v]) for k, v in (exclude_tables or {}).items()}
 
+        self.duplicate_keep_mode = duplicate_keep_mode if duplicate_keep_mode in ('keep_one', 'remove_all') else 'keep_one'
+
     def _get_connection(self) -> pymysql.connections.Connection:
         """
         从连接池获取一个数据库连接。
@@ -416,14 +420,28 @@ class MySQLDeduplicator:
                     if not dry_run:
                         # 分批删除，避免锁表
                         while True:
-                            delete_dup_sql = f"""
-                            DELETE FROM `{database}`.`{table}`
-                            WHERE `{pk_real}` NOT IN (
-                                SELECT `min_id` FROM `{database}`.`{temp_table}`
-                            ) {'AND' if use_time_filter else ''} {f'`{time_col}` >= \'{self._dedup_start_date}\' AND `{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
-                            AND ({' AND '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
-                            LIMIT {self.batch_size}
-                            """
+                            if self.duplicate_keep_mode == 'remove_all':
+                                # 删除所有重复组的所有记录
+                                delete_dup_sql = f"""
+                                DELETE FROM `{database}`.`{table}`
+                                WHERE ({', '.join([f'`{col}`' for col in use_columns])}) IN (
+                                    SELECT {column_list} FROM `{database}`.`{temp_table}`
+                                ) {'AND' if use_time_filter else ''} {f'`{time_col}` >= \'{self._dedup_start_date}\' AND `{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
+                                LIMIT {self.batch_size}
+                                """
+                            else:
+                                # 修正：只删除重复组中不是min_id的行，唯一数据不动
+                                delete_dup_sql = f"""
+                                DELETE FROM `{database}`.`{table}` t
+                                WHERE EXISTS (
+                                    SELECT 1 FROM `{database}`.`{temp_table}` tmp
+                                    WHERE
+                                        {' AND '.join([f't.`{col}` <=> tmp.`{col}`' for col in use_columns])}
+                                        AND t.`{pk_real}` <> tmp.`min_id`
+                                )
+                                {'AND' if use_time_filter else ''} {f't.`{time_col}` >= \'{self._dedup_start_date}\' AND t.`{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
+                                LIMIT {self.batch_size}
+                                """
                             logger.debug('执行删除重复数据SQL', {'sql': delete_dup_sql})
                             cursor.execute(delete_dup_sql)
                             batch_deleted = cursor.rowcount
@@ -431,7 +449,7 @@ class MySQLDeduplicator:
                             conn.commit()
                             if batch_deleted < self.batch_size:
                                 break
-                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组数": dup_count, "实际删除": affected_rows, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns})
+                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组数": dup_count, "实际删除": affected_rows, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns, "去重模式": self.duplicate_keep_mode})
                         # 新增：去重后重排id
                         if reset_id and affected_rows > 0:
                             self._reset_id_column(database, table)
@@ -873,7 +891,6 @@ def main():
     deduplicator.deduplicate_all(dry_run=False, parallel=False, reset_id=False)
 
     # # 指定数据库去重(多线程)
-    # logger.info('调用deduplicate_database')
     # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=True, reset_id=False)
 
     # # 指定表去重(使用特定列)
@@ -883,5 +900,5 @@ def main():
     deduplicator.close()
 
 if __name__ == '__main__':
-    main()
+    # main()
     pass

{mdbq-3.11.6.dist-info → mdbq-3.11.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.11.6
+Version: 3.11.7
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.11.6.dist-info → mdbq-3.11.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=jUdj4-uaa03JUoNnXK_fTx_XQfDwjeFprE71R3ZenRY,18
+mdbq/__version__.py,sha256=KXKzNBZD4M6L-jW29owhKjBycLSiUdGeTf_uNAYvyGI,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -8,7 +8,7 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=HuxLBCXjm6fZrxYE0rdpUCz359WGeqOX0vvg9jTuRY4,24126
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=XSAgt6HqvzDyZSiv4mHli5fA3p3ePn5g3HupqI2cyVo,41444
+mdbq/mysql/deduplicator.py,sha256=dq40YBLVd5ho28pYzVfwm5pA90YA3iN6l9xX4k0Ynds,42808
 mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
 mdbq/mysql/uploader.py,sha256=LxPlAfSNhQbLu-or4wxa-vLjCw5_PIN3ZVoksWUJazQ,61701
@@ -24,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
-mdbq-3.11.6.dist-info/METADATA,sha256=gEcCBqGoPLhugYleGDv1r2YV_lHPL3AnGHD_dHTxY3Q,364
-mdbq-3.11.6.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.11.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.11.6.dist-info/RECORD,,
+mdbq-3.11.7.dist-info/METADATA,sha256=j_0kmOn4tTbk8TY8LqbEZ2OWmJz0-70sUQNwP_N0VCc,364
+mdbq-3.11.7.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.11.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.11.7.dist-info/RECORD,,