mdbq 3.10.9__py3-none-any.whl → 3.10.10__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.10.9'
+VERSION = '3.10.10'

mdbq/mysql/deduplicator.py

@@ -1,5 +1,4 @@
 # -*- coding:utf-8 -*-
-import datetime
 import re
 import time
 from functools import wraps
@@ -7,11 +6,12 @@ import warnings
 import pymysql
 import os
 from mdbq.log import mylogger
-from typing import List, Dict, Optional, Any, Tuple, Set
+from typing import List, Dict, Optional, Any, Tuple
 from dbutils.pooled_db import PooledDB
 import threading
 import concurrent.futures
 from collections import defaultdict
+import sys
 
 
 warnings.filterwarnings('ignore')
@@ -73,27 +73,27 @@ class MySQLDeduplicator:
             max_retries: int = 3,
             retry_interval: int = 5,
             pool_size: int = 5,
-            primary_key: str = 'id'
+            primary_key: str = 'id',
+            date_range: Optional[List[str]] = None,
+            recent_month: Optional[int] = None,
+            date_column: str = '日期',
+            exclude_columns: Optional[List[str]] = None
     ) -> None:
         """
         初始化去重处理器
-
-        :param username: 数据库用户名
-        :param password: 数据库密码
-        :param host: 数据库主机，默认为localhost
-        :param port: 数据库端口，默认为3306
-        :param charset: 字符集，默认为utf8mb4
-        :param max_workers: 最大工作线程数，默认为1(单线程)
-        :param batch_size: 批量处理大小，默认为1000
-        :param skip_system_dbs: 是否跳过系统数据库，默认为True
-        :param max_retries: 最大重试次数
-        :param retry_interval: 重试间隔(秒)
-        :param pool_size: 连接池大小
-        :param primary_key: 主键列名，默认为'id'
+        新增参数:
+        :param date_range: 指定去重的日期区间 [start_date, end_date]，格式'YYYY-MM-DD'
+        :param recent_month: 最近N个月的数据去重（与date_range互斥，优先生效）
+        :param date_column: 时间列名，默认为'日期'
+        :param exclude_columns: 去重时排除的列名列表，默认为['id', '更新时间']
         """
         # 连接池状态标志
         self._closed = False
-
+        logger.debug('初始化MySQLDeduplicator', {
+            'host': host, 'port': port, 'user': username, 'charset': charset,
+            'max_workers': max_workers, 'batch_size': batch_size, 'pool_size': pool_size,
+            'exclude_columns': exclude_columns
+        })
         # 初始化连接池
         self.pool = PooledDB(
             creator=pymysql,
@@ -114,6 +114,33 @@ class MySQLDeduplicator:
         self.retry_interval = retry_interval
         self.primary_key = primary_key
 
+        # 时间范围参数
+        self.date_range = date_range
+        self.recent_month = recent_month
+        self.date_column = date_column
+        self._dedup_start_date = None
+        self._dedup_end_date = None
+        # 不管 exclude_columns 是否传入, 'id' 一定会被排除
+        default_exclude = {'id'}
+        # exclude_columns 不传则排除: ['id', '更新时间']
+        if not exclude_columns:
+            self.exclude_columns = list(default_exclude | {'更新时间'})
+        else:
+            self.exclude_columns = list(set(exclude_columns) | default_exclude)
+        # 解析时间范围
+        if self.date_range and len(self.date_range) == 2:
+            self._dedup_start_date, self._dedup_end_date = self.date_range
+        elif self.recent_month:
+            from datetime import datetime, timedelta
+            today = datetime.today()
+            month = today.month - self.recent_month
+            year = today.year
+            while month <= 0:
+                month += 12
+                year -= 1
+            self._dedup_start_date = f"{year}-{month:02d}-01"
+            self._dedup_end_date = today.strftime("%Y-%m-%d")
+
         # 线程安全控制
         self._lock = threading.Lock()
         self._processing_tables = set()  # 正在处理的表集合
@@ -124,24 +151,25 @@ class MySQLDeduplicator:
     def _get_connection(self) -> pymysql.connections.Connection:
         """从连接池获取连接"""
         if self._closed:
+            logger.error('尝试获取连接但连接池已关闭')
             raise ConnectionError("连接池已关闭")
         try:
             conn = self.pool.connection()
             logger.debug("成功获取数据库连接")
             return conn
         except Exception as e:
-            logger.error(f"获取数据库连接失败: {str(e)}")
+            logger.error(f"获取数据库连接失败: {str(e)}", {'error_type': type(e).__name__})
             raise ConnectionError(f"连接数据库失败: {str(e)}")
 
     @staticmethod
     def _retry_on_failure(func: Any) -> Any:
         """重试装饰器"""
-
         @wraps(func)
         def wrapper(self, *args, **kwargs):
             last_exception = None
             for attempt in range(self.max_retries + 1):
                 try:
+                    logger.debug(f'调用{func.__name__}，第{attempt+1}次尝试', {'args': args, 'kwargs': kwargs})
                     return func(self, *args, **kwargs)
                 except (pymysql.OperationalError, pymysql.InterfaceError) as e:
                     last_exception = e
@@ -149,18 +177,17 @@ class MySQLDeduplicator:
                         wait_time = self.retry_interval * (attempt + 1)
                         logger.warning(
                             f"数据库操作失败，准备重试 (尝试 {attempt + 1}/{self.max_retries})",
-                            {'error': str(e), 'wait_time': wait_time})
+                            {'error': str(e), 'wait_time': wait_time, 'func': func.__name__})
                         time.sleep(wait_time)
                         continue
                 except Exception as e:
                     last_exception = e
-                    logger.error(f"操作失败: {str(e)}", {'error_type': type(e).__name__})
+                    logger.error(f"操作失败: {str(e)}", {'error_type': type(e).__name__, 'func': func.__name__})
                     break
-
             if last_exception:
+                logger.error('重试后依然失败', {'func': func.__name__, 'last_exception': str(last_exception)})
                 raise last_exception
             raise Exception("未知错误")
-
         return wrapper
 
     @_retry_on_failure
@@ -241,104 +268,111 @@ class MySQLDeduplicator:
         """
         if not self._acquire_table_lock(database, table):
             return (0, 0)
-
+        temp_table = None
         try:
-            logger.info(f"开始处理表: {database}.{table}")
-
+            # 获取原始数据总量
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    logger.debug('执行SQL', {'sql': f'SELECT COUNT(*) as cnt FROM `{database}`.`{table}`'})
+                    cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{table}`")
+                    total_count_row = cursor.fetchone()
+                    total_count = total_count_row['cnt'] if total_count_row and 'cnt' in total_count_row else 0
+            logger.info('执行', {"库": database, "表": table, "开始处理数据量": total_count, 'func': sys._getframe().f_code.co_name})
             # 获取实际列名
             all_columns = self._get_table_columns(database, table)
-            if not all_columns:
-                logger.warning(f"表 {database}.{table} 没有有效列(可能只有id列)，跳过")
-                return (0, 0)
-
-            # 使用指定列或所有列
+            logger.debug('获取表列', {'库': database, '表': table, 'all_columns': all_columns})
+            # 检查是否需要按时间范围过滤
+            use_time_filter = False
+            time_col = self.date_column
+            all_columns_lower = [col.lower() for col in all_columns]
+            # 排除exclude_columns
+            exclude_columns_lower = [col.lower() for col in getattr(self, 'exclude_columns', [])]
+            # 统一列名小写做判断
             use_columns = columns or all_columns
-            invalid_columns = set(use_columns) - set(all_columns)
-
+            use_columns = [col for col in use_columns if col.lower() in all_columns_lower and col.lower() not in exclude_columns_lower]
+            invalid_columns = set([col for col in (columns or []) if col.lower() not in all_columns_lower])
             if invalid_columns:
-                logger.warning(
-                    f"表 {database}.{table} 中不存在以下列: {invalid_columns}，使用有效列",
-                    {'invalid_columns': list(invalid_columns)}
-                )
-                use_columns = [col for col in use_columns if col in all_columns]
-
+                logger.warning('不存在的列', {"库": database, "表": table, "不存在以下列": invalid_columns, 'func': sys._getframe().f_code.co_name})
             if not use_columns:
-                logger.error(f"表 {database}.{table} 没有有效的去重列")
+                logger.error('没有有效的去重列', {"库": database, "表": table})
                 return (0, 0)
-
-            # 构建去重SQL
+            # 统一用反引号包裹
             column_list = ', '.join([f'`{col}`' for col in use_columns])
-            # 临时表名限制64字符以内
-            temp_table = f"temp_{table}_dedup_{os.getpid()}"
+            temp_table = f"temp_{table}_dedup_{os.getpid()}_{threading.get_ident()}"
             temp_table = re.sub(r'[^a-zA-Z0-9_]', '_', temp_table)[:60]
             pk = self.primary_key
-            # 校验主键列是否存在
-            if pk not in all_columns and pk != 'id':
-                logger.error(f"表 {database}.{table} 不存在主键列 {pk}")
+            # 主键判断也用小写
+            if pk.lower() not in all_columns_lower and pk != 'id':
+                logger.error('', {"不存在主键列": database, "表": table, "主键列不存在": pk})
                 return (0, 0)
+            # 找到实际主键名
+            pk_real = next((c for c in all_columns if c.lower() == pk.lower()), pk)
+            # 构造where条件
+            where_time = ''
+            if use_time_filter:
+                where_time = f"WHERE `{time_col}` >= '{self._dedup_start_date}' AND `{time_col}` <= '{self._dedup_end_date}'"
             create_temp_sql = f"""
             CREATE TABLE `{database}`.`{temp_table}` AS
-            SELECT MIN(`{pk}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
+            SELECT MIN(`{pk_real}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
             FROM `{database}`.`{table}`
+            {where_time}
             GROUP BY {column_list}
             HAVING COUNT(*) > 1
             """
-
-            delete_dup_sql = f"""
-            DELETE FROM `{database}`.`{table}`
-            WHERE `{pk}` NOT IN (
-                SELECT `min_id` FROM `{database}`.`{temp_table}`
-            ) AND ({' OR '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
-            """
-
             drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
-
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
-                    # 创建临时表统计重复数据
+                    logger.debug('创建临时表SQL', {'sql': create_temp_sql})
                     cursor.execute(create_temp_sql)
+                    logger.debug('统计临时表重复组SQL', {'sql': f'SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`'})
                     cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`")
                     dup_count_row = cursor.fetchone()
                     dup_count = dup_count_row['cnt'] if dup_count_row and 'cnt' in dup_count_row else 0
-
                     if dup_count == 0:
-                        logger.info(f"表 {database}.{table} 没有重复数据")
+                        logger.info('没有重复数据', {"库": database, "表": table, "数据量": total_count, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns})
+                        logger.debug('删除临时表SQL', {'sql': drop_temp_sql})
                         cursor.execute(drop_temp_sql)
                         conn.commit()
                         return (0, 0)
-
-                    logger.info(
-                        f"表 {database}.{table} 发现 {dup_count} 组重复数据",
-                        {'columns': use_columns}
-                    )
-
+                    affected_rows = 0
                     if not dry_run:
-                        # 执行实际删除
-                        cursor.execute(delete_dup_sql)
-                        affected_rows = cursor.rowcount
-                        conn.commit()
-                        logger.info(
-                            f"表 {database}.{table} 已删除 {affected_rows} 行重复数据",
-                            {'columns': use_columns}
-                        )
+                        # 分批删除，避免锁表
+                        while True:
+                            delete_dup_sql = f"""
+                            DELETE FROM `{database}`.`{table}`
+                            WHERE `{pk_real}` NOT IN (
+                                SELECT `min_id` FROM `{database}`.`{temp_table}`
+                            ) {'AND' if use_time_filter else ''} {f'`{time_col}` >= \'{self._dedup_start_date}\' AND `{time_col}` <= \'{self._dedup_end_date}\'' if use_time_filter else ''}
+                            AND ({' AND '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
+                            LIMIT {self.batch_size}
+                            """
+                            logger.debug('执行删除重复数据SQL', {'sql': delete_dup_sql})
+                            cursor.execute(delete_dup_sql)
+                            batch_deleted = cursor.rowcount
+                            affected_rows += batch_deleted
+                            conn.commit()
+                            if batch_deleted < self.batch_size:
+                                break
+                        logger.info('操作删除', {"库": database, "表": table, "数据量": total_count, "重复组数": dup_count, "实际删除": affected_rows, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None, "实际去重列": use_columns})
                     else:
+                        logger.debug('dry_run模式，不执行删除', {"库": database, "表": table, "重复组数": dup_count, "时间范围": [self._dedup_start_date, self._dedup_end_date] if use_time_filter else None})
                         affected_rows = 0
-                        logger.info(
-                            f"[模拟运行] 表 {database}.{table} 将删除 {dup_count} 组重复数据",
-                            {'columns': use_columns}
-                        )
-
-                    # 清理临时表
+                    logger.debug('删除临时表SQL', {'sql': drop_temp_sql})
                     cursor.execute(drop_temp_sql)
                     conn.commit()
-
                     return (dup_count, affected_rows)
-
         except Exception as e:
-            logger.error(
-                f"处理表 {database}.{table} 时出错: {str(e)}",
-                {'error_type': type(e).__name__}
-            )
+            logger.error('异常', {"库": database, "表": table, "异常": str(e), 'func': sys._getframe().f_code.co_name, 'traceback': repr(e)})
+            # 异常时也要清理临时表
+            if temp_table:
+                try:
+                    with self._get_connection() as conn:
+                        with conn.cursor() as cursor:
+                            drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
+                            cursor.execute(drop_temp_sql)
+                            conn.commit()
+                except Exception as drop_e:
+                    logger.error('异常时清理临时表失败', {"库": database, "表": table, "异常": str(drop_e)})
             return (0, 0)
         finally:
             self._release_table_lock(database, table)
@@ -360,17 +394,15 @@ class MySQLDeduplicator:
         :return: (重复行数, 删除行数)
         """
         try:
-            # 检查表是否存在
             if not self._check_table_exists(database, table):
-                logger.warning(f"表 {database}.{table} 不存在，跳过")
+                logger.warning('表不存在', {"库": database, "表": table, "warning": "跳过"})
                 return (0, 0)
-
-            return self._deduplicate_table(database, table, columns, dry_run)
+            logger.info('单表开始', {"库": database, "表": table, "参数": {"指定去重列": columns, "模拟运行": dry_run, '排除列': self.exclude_columns}})
+            result = self._deduplicate_table(database, table, columns, dry_run)
+            logger.info('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result})
+            return result
         except Exception as e:
-            logger.error(
-                f"处理表 {database}.{table} 时发生全局错误: {str(e)}",
-                {'error_type': type(e).__name__}
-            )
+            logger.error('发生全局错误', {"库": database, "表": table, 'func': sys._getframe().f_code.co_name, "发生全局错误": str(e)})
             return (0, 0)
 
     def deduplicate_database(
@@ -392,49 +424,40 @@ class MySQLDeduplicator:
         :return: 字典 {表名: (重复行数, 删除行数)}
         """
         results = {}
-
         try:
-            # 检查数据库是否存在
             if not self._check_database_exists(database):
-                logger.warning(f"数据库 {database} 不存在，跳过")
+                logger.warning('数据库不存在', {"库": database})
                 return results
-
-            # 获取要处理的表
             target_tables = tables or self._get_tables(database)
+            logger.debug('获取目标表', {'库': database, 'tables': target_tables})
             if not target_tables:
-                logger.info(f"数据库 {database} 中没有表，跳过")
+                logger.info('数据库中没有表', {"库": database, "操作": "跳过"})
                 return results
-
-            logger.info(
-                f"开始处理数据库 {database} 中的 {len(target_tables)} 张表",
-                {'tables': target_tables}
-            )
-
+            logger.info('库统计', {"库": database, "表数量": len(target_tables), "表列表": target_tables})
             if parallel and self.max_workers > 1:
-                # 并行处理
+                logger.debug('并行处理表', {'库': database, 'max_workers': self.max_workers})
+                # 使用线程池并行处理
                 with concurrent.futures.ThreadPoolExecutor(
                         max_workers=self.max_workers
                 ) as executor:
                     futures = {}
                     for table in target_tables:
                         columns = columns_map.get(table) if columns_map else None
+                        logger.debug('提交表去重任务', {'库': database, '表': table, 'columns': columns})
                         futures[executor.submit(
                             self.deduplicate_table,
                             database, table, columns, dry_run
                         )] = table
-
                     for future in concurrent.futures.as_completed(futures):
                         table = futures[future]
                         try:
                             dup_count, affected_rows = future.result()
                             results[table] = (dup_count, affected_rows)
                         except Exception as e:
-                            logger.error(
-                                f"处理表 {database}.{table} 时出错: {str(e)}",
-                                {'error_type': type(e).__name__}
-                            )
+                            logger.error('异常', {"库": database, "表": table, "error": str(e), 'traceback': repr(e)})
                             results[table] = (0, 0)
             else:
+                logger.debug('串行处理表', {'库': database})
                 # 串行处理
                 for table in target_tables:
                     columns = columns_map.get(table) if columns_map else None
@@ -442,20 +465,12 @@ class MySQLDeduplicator:
                         database, table, columns, dry_run
                     )
                     results[table] = (dup_count, affected_rows)
-
-            # 统计结果
             total_dup = sum(r[0] for r in results.values())
             total_del = sum(r[1] for r in results.values())
-
-            logger.info(
-                f"数据库 {database} 处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
-                {'results': results}
-            )
-
+            logger.info('单库完成', {"库": database, "重复组数": total_dup, "总删除行数": total_del, "详细结果": results})
             return results
-
         except Exception as e:
-            logger.error(f"处理数据库 {database} 时发生全局错误: {str(e)}", {'error_type': type(e).__name__})
+            logger.error('发生全局错误', {"库": database, 'func': sys._getframe().f_code.co_name, "error": str(e), 'traceback': repr(e)})
             return results
 
     def deduplicate_all(
@@ -477,18 +492,15 @@ class MySQLDeduplicator:
         :return: 嵌套字典 {数据库名: {表名: (重复行数, 删除行数)}}
         """
         all_results = defaultdict(dict)
-
         try:
-            # 获取要处理的数据库
             target_dbs = databases or self._get_databases()
+            logger.debug('获取目标数据库', {'databases': target_dbs})
             if not target_dbs:
-                logger.warning("没有可处理的数据库")
+                logger.warning('没有可处理的数据库')
                 return all_results
-
-            logger.info(f"开始处理 {len(target_dbs)} 个数据库", {'databases': target_dbs})
-
+            logger.info('全局开始', {"数据库数量": len(target_dbs), "数据库列表": target_dbs, "参数": {"模拟运行": dry_run, "并行处理": parallel, '排除列': self.exclude_columns}})
             if parallel and self.max_workers > 1:
-                # 并行处理数据库
+                # 使用线程池并行处理多个数据库
                 with concurrent.futures.ThreadPoolExecutor(
                         max_workers=self.max_workers
                 ) as executor:
@@ -500,14 +512,13 @@ class MySQLDeduplicator:
                             self.deduplicate_database,
                             db, tables, db_columns_map, dry_run, False
                         )] = db
-
                     for future in concurrent.futures.as_completed(futures):
                         db = futures[future]
                         try:
                             db_results = future.result()
                             all_results[db] = db_results
                         except Exception as e:
-                            logger.error(f"处理数据库 {db} 时出错: {str(e)}", {'error_type': type(e).__name__})
+                            logger.error('异常', {"库": db, "error": str(e), 'traceback': repr(e)})
                             all_results[db] = {}
             else:
                 # 串行处理数据库
@@ -518,8 +529,6 @@ class MySQLDeduplicator:
                         db, tables, db_columns_map, dry_run, parallel
                     )
                     all_results[db] = db_results
-
-            # 统计总体结果
             total_dup = sum(
                 r[0] for db in all_results.values()
                 for r in db.values()
@@ -528,16 +537,10 @@ class MySQLDeduplicator:
                 r[1] for db in all_results.values()
                 for r in db.values()
             )
-
-            logger.info(
-                f"所有数据库处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
-                {'total_results': all_results}
-            )
-
+            logger.info('全局完成', {"总重复组数": total_dup, "总删除行数": total_del, "详细结果": dict(all_results)})
             return all_results
-
         except Exception as e:
-            logger.error(f"全局处理时发生错误: {str(e)}", {'error_type': type(e).__name__})
+            logger.error('异常', {"error": str(e), 'traceback': repr(e)})
             return all_results
 
     @_retry_on_failure
@@ -571,8 +574,10 @@ class MySQLDeduplicator:
                 self.pool.close()
                 self._closed = True
                 logger.info("数据库连接池已关闭")
+            else:
+                logger.info('连接池已关闭或不存在')
         except Exception as e:
-            logger.error(f"关闭连接池时出错: {str(e)}", {'error_type': type(e).__name__})
+            logger.error(f"关闭连接池时出错", {'error_type': type(e).__name__, 'error': str(e)})
 
     def __enter__(self) -> 'MySQLDeduplicator':
         return self
@@ -590,13 +595,14 @@ def main():
     )
 
     # 全库去重(单线程)
-    deduplicator.deduplicate_all()
+    deduplicator.deduplicate_all(dry_run=False, parallel=False)
 
     # # 指定数据库去重(多线程)
-    # deduplicator.deduplicate_database('my_db', parallel=True)
+    # logger.info('调用deduplicate_database')
+    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=True)
 
     # # 指定表去重(使用特定列)
-    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
+    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'], dry_run=False)
 
     # 关闭连接
     deduplicator.close()

{mdbq-3.10.9.dist-info → mdbq-3.10.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.9
+Version: 3.10.10
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.10.9.dist-info → mdbq-3.10.10.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=HNkSJG3z_6qz_o8xClGa5PA9tG04IvFuLgr3lvdcfKM,18
+mdbq/__version__.py,sha256=mvvcpn_eYjsZWNgQAvfJdawR8GlNJmr51SxpSdq4Ekc,19
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=zC_w_aVYXwmvfF0Z8iSGMmv5vptF0rP-Dz5zmp0gXTU,19820
 mdbq/aggregation/query_data.py,sha256=fdotW8qdAyDB13p7r3p6AGBkavcHnf6hIvSMtcS7vqE,179875
@@ -9,7 +9,7 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=07sstIeaIQUJXwpMwmxppRI7kW7QwZFnv4Rr3UDlyUs,24133
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=Ingkpaz7Wy6qHxDP4aiy_2c76dSyuIwPF8_pb5dYC48,22542
+mdbq/mysql/deduplicator.py,sha256=sm99eneNO7Br21BH-8vnZW3b7jA3gPF7c9Bvz04YV_g,27759
 mdbq/mysql/mysql.py,sha256=Lfy9PsEdgmdRtcG_UUgegH3bFTJPhByTWkcAYl8G6m0,56788
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
 mdbq/mysql/uploader.py,sha256=3RzslC10pNIYm-0NASicvCHXH0zgUXx7uD1jE21z_OU,64677
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
-mdbq-3.10.9.dist-info/METADATA,sha256=ur4y78xMogVypcNrUAkxFs7JSyZTQPTMpX5YC1wzhCA,364
-mdbq-3.10.9.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.10.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.10.9.dist-info/RECORD,,
+mdbq-3.10.10.dist-info/METADATA,sha256=zIHTb2D1u7ZjwGM-zGMhGJbOTybYgzB30yjPCRBdW5w,365
+mdbq-3.10.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.10.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.10.10.dist-info/RECORD,,