mdbq 3.10.11__py3-none-any.whl → 3.11.0__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.10.11'
+VERSION = '3.11.0'

mdbq/mysql/deduplicator.py

@@ -12,6 +12,7 @@ import threading
 import concurrent.futures
 from collections import defaultdict
 import sys
+from datetime import datetime
 
 
 warnings.filterwarnings('ignore')
@@ -77,15 +78,18 @@ class MySQLDeduplicator:
             date_range: Optional[List[str]] = None,
             recent_month: Optional[int] = None,
             date_column: str = '日期',
-            exclude_columns: Optional[List[str]] = None
+            exclude_columns: Optional[List[str]] = None,
+            exclude_databases: Optional[List[str]] = None,
+            exclude_tables: Optional[Dict[str, List[str]]] = None
     ) -> None:
         """
         初始化去重处理器
-        新增参数:
         :param date_range: 指定去重的日期区间 [start_date, end_date]，格式'YYYY-MM-DD'
         :param recent_month: 最近N个月的数据去重（与date_range互斥，优先生效）
         :param date_column: 时间列名，默认为'日期'
         :param exclude_columns: 去重时排除的列名列表，默认为['id', '更新时间']
+        :param exclude_databases: 排除的数据库名列表
+        :param exclude_tables: 排除的表名字典 {数据库名: [表名, ...]}
         """
         # 连接池状态标志
         self._closed = False
@@ -127,11 +131,28 @@ class MySQLDeduplicator:
             self.exclude_columns = list(default_exclude | {'更新时间'})
         else:
             self.exclude_columns = list(set(exclude_columns) | default_exclude)
-        # 解析时间范围
+        # 解析时间范围并智能校正date_range
         if self.date_range and len(self.date_range) == 2:
-            self._dedup_start_date, self._dedup_end_date = self.date_range
+            try:
+                start, end = self.date_range
+                start_dt = datetime.strptime(start, "%Y-%m-%d")
+                end_dt = datetime.strptime(end, "%Y-%m-%d")
+                if start_dt > end_dt:
+                    logger.warning(
+                        "date_range顺序不正确，自动交换开始和结束日期。",
+                        {"start": start, "end": end}
+                    )
+                    start_dt, end_dt = end_dt, start_dt
+                self._dedup_start_date = start_dt.strftime("%Y-%m-%d")
+                self._dedup_end_date = end_dt.strftime("%Y-%m-%d")
+            except Exception as e:
+                logger.error(
+                    "date_range参数格式错误，应为['YYYY-MM-DD', 'YYYY-MM-DD']，已忽略时间范围。",
+                    {"date_range": self.date_range, "error": str(e)}
+                )
+                self._dedup_start_date = None
+                self._dedup_end_date = None
         elif self.recent_month:
-            from datetime import datetime, timedelta
             today = datetime.today()
             month = today.month - self.recent_month
             year = today.year
@@ -148,8 +169,19 @@ class MySQLDeduplicator:
         # 系统数据库列表
         self.SYSTEM_DATABASES = {'information_schema', 'mysql', 'performance_schema', 'sys'}
 
+        # 排除数据库和表的逻辑
+        self.exclude_databases = set([db.lower() for db in exclude_databases]) if exclude_databases else set()
+        self.exclude_tables = {k.lower(): set([t.lower() for t in v]) for k, v in (exclude_tables or {}).items()}
+
     def _get_connection(self) -> pymysql.connections.Connection:
-        """从连接池获取连接"""
+        """
+        从连接池获取一个数据库连接。
+        
+        Returns:
+            pymysql.connections.Connection: 数据库连接对象。
+        Raises:
+            ConnectionError: 如果连接池已关闭或获取连接失败。
+        """
         if self._closed:
             logger.error('尝试获取连接但连接池已关闭')
             raise ConnectionError("连接池已关闭")
@@ -163,7 +195,17 @@ class MySQLDeduplicator:
 
     @staticmethod
     def _retry_on_failure(func: Any) -> Any:
-        """重试装饰器"""
+        """
+        装饰器：为数据库操作方法提供自动重试机制。
+        仅捕获pymysql的连接相关异常，重试指定次数后抛出最后一次异常。
+        
+        Args:
+            func (Any): 被装饰的函数。
+        Returns:
+            Any: 被装饰函数的返回值。
+        Raises:
+            Exception: 多次重试后依然失败时抛出。
+        """
         @wraps(func)
         def wrapper(self, *args, **kwargs):
             last_exception = None
@@ -192,21 +234,31 @@ class MySQLDeduplicator:
 
     @_retry_on_failure
     def _get_databases(self) -> List[str]:
-        """获取所有非系统数据库列表"""
+        """
+        获取所有非系统数据库列表，排除exclude_databases。
+        
+        Returns:
+            List[str]: 数据库名列表。
+        """
         sql = "SHOW DATABASES"
-
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
                 cursor.execute(sql)
                 all_dbs = [row['Database'] for row in cursor.fetchall()]
-
-                if self.skip_system_dbs:
-                    return [db for db in all_dbs if db.lower() not in self.SYSTEM_DATABASES]
-                return all_dbs
+                # 排除系统库和用户指定的库
+                filtered = [db for db in all_dbs if db.lower() not in self.SYSTEM_DATABASES and db.lower() not in self.exclude_databases] if self.skip_system_dbs else [db for db in all_dbs if db.lower() not in self.exclude_databases]
+                return filtered
 
     @_retry_on_failure
     def _get_tables(self, database: str) -> List[str]:
-        """获取指定数据库的所有表"""
+        """
+        获取指定数据库的所有表名。
+        
+        Args:
+            database (str): 数据库名。
+        Returns:
+            List[str]: 表名列表。
+        """
         sql = "SHOW TABLES"
 
         with self._get_connection() as conn:
@@ -217,7 +269,15 @@ class MySQLDeduplicator:
 
     @_retry_on_failure
     def _get_table_columns(self, database: str, table: str) -> List[str]:
-        """获取表的列名(排除主键列)"""
+        """
+        获取指定表的所有列名（排除主键列）。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+        Returns:
+            List[str]: 列名列表。
+        """
         sql = """
         SELECT COLUMN_NAME 
         FROM INFORMATION_SCHEMA.COLUMNS 
@@ -232,7 +292,15 @@ class MySQLDeduplicator:
                         if row['COLUMN_NAME'].lower() != self.primary_key.lower()]
 
     def _acquire_table_lock(self, database: str, table: str) -> bool:
-        """获取表处理锁，防止并发处理同一张表"""
+        """
+        获取表处理锁，防止并发处理同一张表。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+        Returns:
+            bool: 是否成功获取锁。
+        """
         key = f"{database}.{table}"
 
         with self._lock:
@@ -243,7 +311,13 @@ class MySQLDeduplicator:
             return True
 
     def _release_table_lock(self, database: str, table: str) -> None:
-        """释放表处理锁"""
+        """
+        释放表处理锁。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+        """
         key = f"{database}.{table}"
 
         with self._lock:
@@ -258,13 +332,15 @@ class MySQLDeduplicator:
             dry_run: bool = False
     ) -> Tuple[int, int]:
         """
-        执行单表去重
-
-        :param database: 数据库名
-        :param table: 表名
-        :param columns: 用于去重的列(为None时使用所有列)
-        :param dry_run: 是否模拟运行(只统计不实际删除)
-        :return: (重复行数, 删除行数)
+        执行单表去重。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
+            dry_run (bool): 是否为模拟运行（只统计不实际删除）。
+        Returns:
+            Tuple[int, int]: (重复组数, 实际删除行数)。
         """
         if not self._acquire_table_lock(database, table):
             return (0, 0)
@@ -385,14 +461,19 @@ class MySQLDeduplicator:
             dry_run: bool = False
     ) -> Tuple[int, int]:
         """
-        对指定表进行去重
-
-        :param database: 数据库名
-        :param table: 表名
-        :param columns: 用于去重的列(为None时使用所有列)
-        :param dry_run: 是否模拟运行(只统计不实际删除)
-        :return: (重复行数, 删除行数)
+        对指定表进行去重。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+            columns (Optional[List[str]]): 用于去重的列名列表（为None时使用所有列）。
+            dry_run (bool): 是否为模拟运行（只统计不实际删除）。
+        Returns:
+            Tuple[int, int]: (重复组数, 实际删除行数)。
         """
+        if database.lower() in self.exclude_tables and table.lower() in self.exclude_tables[database.lower()]:
+            logger.info('表被排除', {"库": database, "表": table, "操作": "跳过"})
+            return (0, 0)
         try:
             if not self._check_table_exists(database, table):
                 logger.warning('表不存在', {"库": database, "表": table, "warning": "跳过"})
@@ -414,14 +495,16 @@ class MySQLDeduplicator:
             parallel: bool = False
     ) -> Dict[str, Tuple[int, int]]:
         """
-        对指定数据库的所有表进行去重
-
-        :param database: 数据库名
-        :param tables: 要处理的表列表(为None时处理所有表)
-        :param columns_map: 各表使用的去重列 {表名: [列名]}
-        :param dry_run: 是否模拟运行
-        :param parallel: 是否并行处理
-        :return: 字典 {表名: (重复行数, 删除行数)}
+        对指定数据库的所有表进行去重。
+        
+        Args:
+            database (str): 数据库名。
+            tables (Optional[List[str]]): 要处理的表列表（为None时处理所有表）。
+            columns_map (Optional[Dict[str, List[str]]]): 各表使用的去重列 {表名: [列名]}。
+            dry_run (bool): 是否为模拟运行。
+            parallel (bool): 是否并行处理。
+        Returns:
+            Dict[str, Tuple[int, int]]: {表名: (重复组数, 实际删除行数)}。
         """
         results = {}
         try:
@@ -429,6 +512,8 @@ class MySQLDeduplicator:
                 logger.warning('数据库不存在', {"库": database})
                 return results
             target_tables = tables or self._get_tables(database)
+            exclude_tbls = self.exclude_tables.get(database.lower(), set())
+            target_tables = [t for t in target_tables if t.lower() not in exclude_tbls]
             logger.debug('获取目标表', {'库': database, 'tables': target_tables})
             if not target_tables:
                 logger.info('数据库中没有表', {"库": database, "操作": "跳过"})
@@ -482,18 +567,21 @@ class MySQLDeduplicator:
             parallel: bool = False
     ) -> Dict[str, Dict[str, Tuple[int, int]]]:
         """
-        对所有数据库进行去重
-
-        :param databases: 要处理的数据库列表(为None时处理所有非系统数据库)
-        :param tables_map: 各数据库要处理的表 {数据库名: [表名]}
-        :param columns_map: 各表使用的去重列 {数据库名: {表名: [列名]}}
-        :param dry_run: 是否模拟运行
-        :param parallel: 是否并行处理
-        :return: 嵌套字典 {数据库名: {表名: (重复行数, 删除行数)}}
+        对所有数据库进行去重。
+        
+        Args:
+            databases (Optional[List[str]]): 要处理的数据库列表。如果为 None，则处理所有非系统数据库。
+            tables_map (Optional[Dict[str, List[str]]]): 指定每个数据库要处理的表，格式为 {数据库名: [表名, ...]}。如果为 None，则处理所有表。
+            columns_map (Optional[Dict[str, Dict[str, List[str]]]]): 指定每个表去重时使用的列，格式为 {数据库名: {表名: [列名, ...]}}。如果为 None，则使用所有列。
+            dry_run (bool): 是否为模拟运行模式。为 True 时只统计重复行数，不实际删除。
+            parallel (bool): 是否并行处理多个数据库。为 True 时使用线程池并发处理。
+        Returns:
+            Dict[str, Dict[str, Tuple[int, int]]]: 嵌套字典，格式为 {数据库名: {表名: (重复组数, 实际删除行数)}}。
         """
-        all_results = defaultdict(dict)
+        all_results: Dict[str, Dict[str, Tuple[int, int]]] = defaultdict(dict)
         try:
-            target_dbs = databases or self._get_databases()
+            target_dbs: List[str] = databases or self._get_databases()
+            target_dbs = [db for db in target_dbs if db.lower() not in self.exclude_databases]
             logger.debug('获取目标数据库', {'databases': target_dbs})
             if not target_dbs:
                 logger.warning('没有可处理的数据库')
@@ -504,7 +592,7 @@ class MySQLDeduplicator:
                 with concurrent.futures.ThreadPoolExecutor(
                         max_workers=self.max_workers
                 ) as executor:
-                    futures = {}
+                    futures: Dict[concurrent.futures.Future, str] = {}
                     for db in target_dbs:
                         tables = tables_map.get(db) if tables_map else None
                         db_columns_map = columns_map.get(db) if columns_map else None
@@ -545,7 +633,14 @@ class MySQLDeduplicator:
 
     @_retry_on_failure
     def _check_database_exists(self, database: str) -> bool:
-        """检查数据库是否存在"""
+        """
+        检查数据库是否存在。
+        
+        Args:
+            database (str): 数据库名。
+        Returns:
+            bool: 数据库是否存在。
+        """
         sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
 
         with self._get_connection() as conn:
@@ -555,7 +650,15 @@ class MySQLDeduplicator:
 
     @_retry_on_failure
     def _check_table_exists(self, database: str, table: str) -> bool:
-        """检查表是否存在"""
+        """
+        检查表是否存在。
+        
+        Args:
+            database (str): 数据库名。
+            table (str): 表名。
+        Returns:
+            bool: 表是否存在。
+        """
         sql = """
         SELECT TABLE_NAME 
         FROM INFORMATION_SCHEMA.TABLES 
@@ -568,7 +671,12 @@ class MySQLDeduplicator:
                 return bool(cursor.fetchone())
 
     def close(self) -> None:
-        """关闭连接池"""
+        """
+        关闭连接池。
+        
+        Returns:
+            None
+        """
         try:
             if hasattr(self, 'pool') and self.pool and not self._closed:
                 self.pool.close()
@@ -580,9 +688,25 @@ class MySQLDeduplicator:
             logger.error(f"关闭连接池时出错", {'error_type': type(e).__name__, 'error': str(e)})
 
     def __enter__(self) -> 'MySQLDeduplicator':
+        """
+        上下文管理器进入方法。
+        
+        Returns:
+            MySQLDeduplicator: 实例自身。
+        """
         return self
 
     def __exit__(self, exc_type: Optional[type], exc_val: Optional[BaseException], exc_tb: Optional[Any]) -> None:
+        """
+        上下文管理器退出方法，自动关闭连接池。
+        
+        Args:
+            exc_type (Optional[type]): 异常类型。
+            exc_val (Optional[BaseException]): 异常值。
+            exc_tb (Optional[Any]): 异常追踪。
+        Returns:
+            None
+        """
         self.close()
 
 

mdbq/mysql/uploader.py

@@ -346,8 +346,8 @@ class MySQLUploader:
             logger.error('无效的标识符', {'标识符': identifier})
             raise ValueError(f"无效的标识符: `{identifier}`")
         if not self.case_sensitive:
-            identifier = identifier.lower()
-        cleaned = re.sub(r'[^-\w\u4e00-\u9fff$]', '_', identifier)
+            
+        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '_', identifier)
         cleaned = re.sub(r'_+', '_', cleaned).strip('_')
         if not cleaned:
             logger.error('无法清理异常标识符', {'原始标识符': identifier})

{mdbq-3.10.11.dist-info → mdbq-3.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.11
+Version: 3.11.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.10.11.dist-info → mdbq-3.11.0.dist-info}/RECORD

@@ -1,7 +1,6 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=9LS36YFJz4Ep6F-NYjHbq6pXXVJAhfLpffZVlSwjaSI,19
+mdbq/__version__.py,sha256=vrerkwEDL5E5nSSdgLRAwkxPSJb14-hLfjbjj6J7G3I,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/optimize.py,sha256=zC_w_aVYXwmvfF0Z8iSGMmv5vptF0rP-Dz5zmp0gXTU,19820
 mdbq/aggregation/query_data.py,sha256=fdotW8qdAyDB13p7r3p6AGBkavcHnf6hIvSMtcS7vqE,179875
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
@@ -9,10 +8,10 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=07sstIeaIQUJXwpMwmxppRI7kW7QwZFnv4Rr3UDlyUs,24133
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=sm99eneNO7Br21BH-8vnZW3b7jA3gPF7c9Bvz04YV_g,27759
+mdbq/mysql/deduplicator.py,sha256=ibmxpzenhPgT_ei61TjQB2ZxYs9ztkG_ygbLSa8RIlM,32990
 mdbq/mysql/mysql.py,sha256=Lfy9PsEdgmdRtcG_UUgegH3bFTJPhByTWkcAYl8G6m0,56788
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
-mdbq/mysql/uploader.py,sha256=CyNrNTGBvxwfG5NygG1tnsgVTqfKw_U5BewFZvObhJ0,66485
+mdbq/mysql/uploader.py,sha256=6b1NXGtQnhpSWMXnw0ai07ejS1GzMUZWzcjG8G68pbY,66451
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
-mdbq-3.10.11.dist-info/METADATA,sha256=GrJ6G6ZtBfCsHMzbSkBzWYn_GIPa_W_rAI6tSLK6Pjs,365
-mdbq-3.10.11.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.10.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.10.11.dist-info/RECORD,,
+mdbq-3.11.0.dist-info/METADATA,sha256=go0OKEPPSfRYS3OQJ0A2bJP1FPNCbtAtp-LHZlrh9NM,364
+mdbq-3.11.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.11.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.11.0.dist-info/RECORD,,

mdbq/aggregation/optimize.py

@@ -1,475 +0,0 @@
-# -*- coding:utf-8 -*-
-import pymysql
-import logging
-from typing import List, Optional, Dict
-import time
-import re
-import os
-import hashlib
-from dbutils.pooled_db import PooledDB
-from mdbq.log import spider_logging
-from mdbq.config import config
-import threading
-import queue
-
-dir_path = os.path.expanduser("~")
-config_file = os.path.join(dir_path, 'spd.txt')
-my_cont = config.read_config(config_file)
-username, password, port = my_cont['username'], my_cont['password'], my_cont['port']
-host = '127.0.0.1'
-logger = spider_logging.setup_logging(reMoveOldHandler=True, filename='optimize.log')
-
-
-class MySQLDeduplicator:
-
-    def __init__(self, host: str, username: str, password: str, port: int = 3306):
-        self.pool = PooledDB(
-            creator=pymysql,
-            maxconnections=10,  # 最大连接数
-            mincached=2,  # 初始化空闲连接数
-            maxcached=5,  # 空闲连接最大缓存数
-            blocking=True,
-            host=host,
-            port=int(port),
-            user=username,
-            password=password,
-            ping=1,
-            charset='utf8mb4',
-            cursorclass=pymysql.cursors.DictCursor
-        )
-        self.set_typ = {
-            '日期': 'date',
-            '更新时间': 'timestamp',
-        }
-        self.tables_to_reset = queue.Queue()  # 线程安全队列
-        self.delay_time = 120  # 延迟重置自增 id
-        self.lock = threading.Lock()  # 用于关键操作同步
-
-    def get_table_in_databases(self, db_list=None, reset_id=False):
-        """
-        reset_id: 是否重置自增 id
-        """
-        if not db_list:
-            return
-        connection = self.get_connection()
-        res = []
-        for db_name in db_list:
-            try:
-                with connection.cursor() as cursor:
-                    cursor.execute(f"USE `{db_name}`")
-                    cursor.execute("SHOW TABLES")
-                    tables = cursor.fetchall()
-                    for index, item in enumerate(tables):
-                        res.append(
-                            {
-                                'db_name': db_name,
-                                'table_name': item.get(f'Tables_in_{db_name}', ''),
-                                'reset_id': reset_id,
-                            }
-                        )
-            except:
-                pass
-        connection.close()
-        return res
-
-    def deduplicate(
-            self,
-            tables_list: List[Dict],
-            order_column: str = "更新时间",
-            order_direction: str = "DESC",
-            batch_size: int = 10000,
-            id_column: str = "id",
-            recent_months: Optional[int] = None
-    ) -> bool:
-        """
-        执行多表去重操作
-        :param tables_list: 目标表配置列表，每个元素为字典，包含db_name, table_name, unique_keys（可选）, reset_id（可选）
-        :param order_column: 排序字段
-        :param order_direction: 排序方向 (ASC/DESC)
-        :param batch_size: 批量删除批次大小
-        :param id_column: 自增列名称
-        :return: 是否全部成功
-        """
-        if recent_months is not None and (not isinstance(recent_months, int) or recent_months < 1):
-            logger.error("recent_months必须为None或正整数")
-            return False
-        for table_config in tables_list:
-            config = {
-                'order_column': order_column,
-                'order_direction': order_direction,
-                'batch_size': batch_size,
-                'id_column': id_column,
-                'reset_id': table_config.get('reset_id', False),  # 处理默认值
-                'unique_keys': table_config.get('unique_keys', None),
-                'recent_months': recent_months,
-            }
-            config.update(table_config)
-            self._deduplicate_single_table(**config)
-
-    def _deduplicate_single_table(
-            self,
-            db_name: str,
-            table_name: str,
-            unique_keys: Optional[List[str]],
-            order_column: str,
-            order_direction: str,
-            batch_size: int,
-            reset_id: bool,
-            id_column: str,
-            recent_months: Optional[int] = None
-    ):
-        """单表去重逻辑"""
-
-        # 获取数据库连接并检查有效性
-        connection = self.get_connection(db_name=db_name)
-        if not connection:
-            logger.error(f"连接数据库失败: {db_name}")
-            return False
-
-        temp_suffix = hashlib.md5(f"{table_name}{time.time()}".encode()).hexdigest()[:8]
-        temp_table = f"temp_{temp_suffix}"
-
-        try:
-            # 版本检查在check_db内部
-            if not self.check_db(db_name, table_name):
-                return False
-
-            with connection.cursor() as cursor:
-                # 主键重复检查
-                try:
-                    cursor.execute(f"""
-                        SELECT COUNT(*) AS total, 
-                               COUNT(DISTINCT `{id_column}`) AS distinct_count 
-                        FROM `{table_name}`
-                    """)
-                except pymysql.err.InternalError as e:
-                    if e.args[0] == pymysql.constants.ER.BAD_FIELD_ERROR:
-                        logger.warning(f"{db_name}/{table_name} 跳过主键检查（无{id_column}列）")
-                    else:
-                        raise
-                else:
-                    res = cursor.fetchone()
-                    if res['total'] != res['distinct_count']:
-                        logger.error(f"{db_name}/{table_name} 主键重复: {id_column}")
-                        return False
-
-                all_columns = self._get_table_columns(db_name, table_name)
-                # 自动生成unique_keys逻辑
-                if not unique_keys:
-                    exclude_set = {id_column.lower(), order_column.lower()}
-
-                    if not all_columns:
-                        logger.error(f"{db_name}/{table_name} 无法获取表列信息")
-                        return False
-
-                    # 排除id_column和order_column
-                    unique_keys = [
-                        col for col in all_columns
-                        if col.lower() not in exclude_set
-                           and col != id_column  # 额外确保大小写兼容
-                           and col != order_column
-                    ]
-                    # 检查剩余列是否有效
-                    if not unique_keys:
-                        unique_keys = all_columns
-                        logger.warning(f"{db_name}/{table_name} 使用全列作为唯一键: {all_columns}")
-                        return False
-                    # logger.info(f"自动生成unique_keys: {unique_keys}")
-                else:
-                    if not self._validate_columns(db_name, table_name, unique_keys):
-                        logger.error(f"{db_name}/{table_name} unique_keys中存在无效列名")
-                        return False
-
-                # 动态生成临时表名
-                partition_clause = ', '.join([f'`{col}`' for col in unique_keys])
-
-                # 使用参数化查询创建临时表
-                if self._validate_columns(db_name, table_name, [order_column]):
-                    order_clause = f"ORDER BY `{order_column}` {order_direction}" if order_column else ""
-                else:
-                    order_clause = ''
-
-                # 时间过滤
-                where_clause = ""
-                query_params = []
-                date_column_exists = '日期' in all_columns
-                if recent_months and recent_months > 0 and date_column_exists:
-                    where_clause = "WHERE `日期` >= DATE_SUB(CURDATE(), INTERVAL %s MONTH)"
-                    query_params.append(recent_months)
-                elif recent_months and not date_column_exists:
-                    logger.warning(f"{db_name}/{table_name} 忽略recent_months参数（无日期列）")
-
-                create_temp_sql = f"""
-                            CREATE TEMPORARY TABLE `{temp_table}` AS
-                            SELECT tmp_id FROM (
-                                SELECT `{id_column}` AS tmp_id,
-                                ROW_NUMBER() OVER (
-                                    PARTITION BY {partition_clause or '1'}
-                                    {order_clause}
-                                ) AS row_num
-                                FROM `{table_name}`
-                                {where_clause}
-                            ) t WHERE row_num > 1;
-                        """
-                cursor.execute(create_temp_sql, query_params)
-
-                logger.info(f'{db_name}/{table_name}  执行排重任务')
-                # 批量删除优化
-                iteration = 0
-                total_deleted = 0
-                while True and iteration < 10000:
-                    iteration += 1
-                    # 获取并删除临时表中的数据，避免重复处理
-                    cursor.execute(f"""
-                            SELECT tmp_id 
-                            FROM `{temp_table}` 
-                            LIMIT %s 
-                            FOR UPDATE;
-                        """, (batch_size,))
-                    batch = cursor.fetchall()
-                    if not batch:
-                        break
-                    ids = [str(row['tmp_id']) for row in batch]
-                    placeholder = ','.join(['%s'] * len(ids))
-
-                    if ids:
-                        try:
-                            # 删除主表数据
-                            cursor.execute(f"DELETE FROM `{table_name}` WHERE `{id_column}` IN ({placeholder})", ids)
-
-                            # 删除临时表中已处理的记录
-                            cursor.execute(f"DELETE FROM `{temp_table}` WHERE tmp_id IN ({placeholder})", ids)
-                        except pymysql.err.InternalError as e:
-                            if e.args[0] == pymysql.constants.ER.BAD_FIELD_ERROR:
-                                logger.error(f"{db_name}/{table_name}  无法通过 {id_column} 删除记录，请检查列存在性")
-                                return False
-                            raise
-
-                    total_deleted += cursor.rowcount
-                    connection.commit()
-                    logger.info(f"{db_name}/{table_name}  执行去重, 删除记录数: {total_deleted}")
-
-                if total_deleted > 0:
-                    logger.info(f"{db_name}/{table_name}  删除记录数总计: {total_deleted}")
-
-                # 线程安全操作队列
-                if reset_id:
-                    if not self._validate_columns(db_name, table_name, [id_column]):
-                        return True
-
-                    with self.lock:
-                        self.tables_to_reset.put((db_name, table_name, id_column))
-                    logger.info(f"{db_name}/{table_name} -> {self.delay_time}秒后重置自增id")
-                    threading.Timer(self.delay_time, self.delayed_reset_auto_increment).start()
-
-                return True
-        except Exception as e:
-            logger.error(f"{db_name}/{table_name} 去重操作异常: {e}", exc_info=True)
-            connection.rollback()
-            return False
-        finally:
-            with connection.cursor() as cursor:
-                cursor.execute(f"DROP TEMPORARY TABLE IF EXISTS `{temp_table}`")
-            connection.close()
-
-    def _get_table_columns(self, db_name: str, table_name: str) -> List[str]:
-        """获取表的列"""
-        try:
-            connection = self.get_connection(db_name=db_name)
-            with connection.cursor() as cursor:
-                cursor.execute(f"SHOW COLUMNS FROM `{table_name}`")
-                return [row["Field"] for row in cursor.fetchall()]
-        except pymysql.Error as e:
-            logging.error(f"{db_name}/{table_name} 获取列失败: {e}")
-            return []
-
-    def check_db(self, db_name: str, table_name: str) -> bool:
-        """数据库检查"""
-        try:
-            with self.get_connection() as conn:
-                with conn.cursor() as cursor:
-                    # 获取MySQL版本
-                    version = self._check_mysql_version(cursor)
-                    collation = 'utf8mb4_0900_ai_ci' if version >= 8.0 else 'utf8mb4_general_ci'
-
-                    # 创建数据库
-                    cursor.execute(f"""
-                        CREATE DATABASE IF NOT EXISTS `{db_name}` 
-                        CHARACTER SET utf8mb4 COLLATE {collation}
-                    """)
-                    conn.commit()
-
-                    # 切换数据库
-                    cursor.execute(f"USE `{db_name}`")
-
-                    # 检查表是否存在
-                    if not self._table_exists(cursor, table_name):
-                        self._create_table(cursor, table_name)
-                        conn.commit()
-                    return True
-        except Exception as e:
-            logger.error(f"{db_name}/{table_name} 数据库检查失败: {e}")
-            return False
-
-    def get_connection(self, db_name=None):
-        """从连接池获取连接"""
-        for _ in range(10):
-            try:
-                if db_name:
-                    connection = self.pool.connection()
-                    with connection.cursor() as cursor:
-                        cursor.execute(f'use {db_name};')
-                        return connection
-
-                return self.pool.connection()
-            except pymysql.Error as e:
-                logger.error(f"{db_name}  获取连接失败: {e}, 30秒后重试...")
-                time.sleep(30)
-        logger.error(f"{host}: {port}  数据库连接失败，已达最大重试次数")
-        return None
-
-    def _validate_identifier(self, name: str) -> bool:
-        """更严格的对象名验证（符合MySQL规范）"""
-        return re.match(r'^[\w$]+$', name) and len(name) <= 64
-
-    def _validate_columns(self, db_name: str, table_name: str, columns: List[str]) -> bool:
-        """验证列是否存在"""
-        if not all(self._validate_identifier(col) for col in columns):
-            return False
-        try:
-            connection = self.get_connection(db_name=db_name)
-            with connection.cursor() as cursor:
-                cursor.execute(f"SHOW COLUMNS FROM `{table_name}`")
-                existing_columns = {col['Field'] for col in cursor.fetchall()}
-                return all(col in existing_columns for col in columns)
-        except pymysql.Error as e:
-            logging.error(f"{db_name}/{table_name} 列验证失败: {e}")
-            return False
-
-    def _check_mysql_version(self, cursor) -> float:
-        """通过传入游标检查版本"""
-        cursor.execute("SELECT VERSION()")
-        return float(cursor.fetchone()['VERSION()'][:3])
-
-    def _table_exists(self, cursor, table_name: str) -> bool:
-        cursor.execute("SHOW TABLES LIKE %s", (table_name,))
-        return cursor.fetchone() is not None
-
-    def _create_table(self, cursor, table_name: str):
-        """安全建表逻辑"""
-        columns = ["`id` INT AUTO_INCREMENT PRIMARY KEY"]
-        for cn, ct in self.set_typ.items():
-            col_def = f"`{cn}` {ct.upper()} NOT NULL DEFAULT "
-            if 'INT' in ct:
-                col_def += '0'
-            elif 'TIMESTAMP' in ct:
-                col_def += 'CURRENT_TIMESTAMP'
-            else:
-                col_def += "''"
-            columns.append(col_def)
-        cursor.execute(f"""
-            CREATE TABLE `{table_name}` (
-                {', '.join(columns)}
-            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
-        """)
-
-    def delayed_reset_auto_increment(self):
-        """线程安全的自增ID重置"""
-        while not self.tables_to_reset.empty():
-            try:
-                item = self.tables_to_reset.get_nowait()
-                self._safe_reset_auto_increment(*item)
-            except queue.Empty:
-                break
-
-    def _safe_reset_auto_increment(self, db_name: str, table_name: str, id_column: str):
-        """安全重置自增ID"""
-        with self.get_connection(db_name) as conn:
-            try:
-                with conn.cursor() as cursor:
-                    cursor.execute("START TRANSACTION")
-                    temp_table = f"reset_{hashlib.md5(table_name.encode()).hexdigest()[:8]}"
-                    backup_table = f"{table_name}_backup_{int(time.time())}"
-                    cursor.execute(f"CREATE TABLE `{temp_table}` LIKE `{table_name}`")
-                    cursor.execute(f"ALTER TABLE `{temp_table}` MODIFY COLUMN `{id_column}` INT NOT NULL")
-                    columns = self._get_table_columns(db_name, table_name)
-                    if id_column not in columns:
-                        logger.error(f"列 {id_column} 不存在于表 {table_name}")
-                        return False
-                    columns.remove(id_column)
-                    columns_str = ', '.join([f'`{col}`' for col in columns])
-                    insert_sql = f"""
-                        INSERT INTO `{temp_table}` (`{id_column}`, {columns_str})
-                        SELECT ROW_NUMBER() OVER (ORDER BY `{id_column}`), {columns_str}
-                        FROM `{table_name}` ORDER BY `{id_column}`
-                    """
-                    cursor.execute(insert_sql)
-                    cursor.execute(f"RENAME TABLE `{table_name}` TO `{backup_table}`, `{temp_table}` TO `{table_name}`")
-                    cursor.execute(f"ALTER TABLE `{table_name}` MODIFY COLUMN `{id_column}` INT AUTO_INCREMENT")
-                    cursor.execute(f"SELECT MAX(`{id_column}`) + 1 AS next_id FROM `{table_name}`")
-                    next_id = cursor.fetchone()['next_id'] or 1
-                    cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = {next_id}")
-                    cursor.execute(f"DROP TABLE IF EXISTS `{backup_table}`")
-                    cursor.execute(f"DROP TEMPORARY TABLE IF EXISTS `{temp_table}`")
-                    cursor.execute("COMMIT")
-                    logger.info(f'{db_name}/{table_name} 已重置自增id')
-            except Exception as e:
-                logger.error(f"{db_name}/{table_name} 重置自增id失败: {e}")
-                cursor.execute("ROLLBACK")
-                return False
-            finally:
-                conn.close()
-
-
-def main():
-    op = MySQLDeduplicator(
-        host=host,
-        username=username,
-        password=password,
-        port=port
-    )
-    op.delay_time = 600
-    # tables_list = [
-    #     {
-    #         'db_name': "测试库",
-    #         'table_name': "测试库2",
-    #         'reset_id': True,  # 可选, 默认 False
-    #         # 'unique_keys': ["日期", "店铺名称", "商品id"]
-    #     }
-    # ]
-    db_list = [
-        "京东数据3",
-        "属性设置3",
-        "推广数据2",
-        "推广数据_圣积天猫店",
-        "推广数据_淘宝店",
-        "推广数据_奥莱店",
-        "爱库存2",
-        "生意参谋3",
-        "生意经3",
-        "达摩盘3",
-        '人群画像2',
-        '商品人群画像2',
-        '市场数据3',
-        # '数据银行2'
-        # '回传数据',
-        # '大模型库',
-        '安全组',
-        # '视频数据',
-        # '聚合数据',
-        '数据引擎2'
-    ]
-    tables_list = op.get_table_in_databases(db_list=db_list, reset_id=False)
-    op.deduplicate(
-        order_column = "更新时间",
-        order_direction = "DESC",
-        batch_size = 1000,
-        id_column = "id",
-        tables_list=tables_list,
-        recent_months=3,
-    )
-    logger.info(f'全部任务完成')
-
-
-if __name__ == "__main__":
-    main()