mdbq 3.12.3__py3-none-any.whl → 3.12.4__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.12.3'
+VERSION = '3.12.4'

mdbq/mysql/deduplicator.py

@@ -16,12 +16,11 @@ from datetime import datetime
 import uuid
 from contextlib import contextmanager
 
-
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
     name='deduplicator',
     logging_mode='file',
-    log_level='debug',
+    log_level='info',
     log_file='deduplicator.log',
     log_format='json',
     max_log_size=50,
@@ -72,7 +71,8 @@ class MySQLDeduplicator:
             exclude_columns: Optional[List[str]] = None,
             exclude_databases: Optional[List[str]] = None,
             exclude_tables: Optional[Dict[str, List[str]]] = None,
-            duplicate_keep_mode: str = 'keep_one'
+            duplicate_keep_mode: str = 'keep_one',
+            keep_order: str = 'min'
     ) -> None:
         """
         初始化去重处理器
@@ -178,6 +178,7 @@ class MySQLDeduplicator:
         self.exclude_tables = {k.lower(): set(t.lower() for t in v) for k, v in (exclude_tables or {}).items()}
 
         self.duplicate_keep_mode = duplicate_keep_mode if duplicate_keep_mode in ('keep_one', 'remove_all') else 'keep_one'
+        self.keep_order = keep_order if keep_order in ('min', 'max') else 'min'
 
     def _get_connection(self) -> pymysql.connections.Connection:
         """
@@ -442,18 +443,31 @@ class MySQLDeduplicator:
 
                 # 用Python查找重复
                 if use_python_dedup:
+                    # 判断分组字段是否有“更新时间”
+                    has_update_time = any(col == '更新时间' for col in use_columns)
                     select_cols = f'`{pk_real}`,' + ','.join([f'`{col}`' for col in use_columns])
-                    select_where = f"WHERE `{time_col}` = '{date_val}'"
+                    if has_update_time:
+                        select_cols += ',`更新时间`'
+                    select_where = f"WHERE `{time_col}` = '{date_val}'" if date_val else ''
                     grouped = defaultdict(list)
                     for row in self._row_generator(database, table, select_cols, select_where, self.batch_size):
                         key = tuple(row[col] for col in use_columns)
-                        grouped[key].append(row[pk_real])
+                        grouped[key].append(row)
                     dup_count = 0
                     del_ids = []
                     for ids in grouped.values():
                         if len(ids) > 1:
                             dup_count += 1
-                            del_ids.extend(ids[1:])
+                            if has_update_time:
+                                # 按更新时间最大保留
+                                keep_row = max(ids, key=lambda x: x.get('更新时间') or '')
+                            else:
+                                # 按id保留
+                                if self.keep_order == 'max':
+                                    keep_row = max(ids, key=lambda x: x[pk_real])
+                                else:
+                                    keep_row = min(ids, key=lambda x: x[pk_real])
+                            del_ids.extend([r[pk_real] for r in ids if r[pk_real] != keep_row[pk_real]])
                     affected_rows = 0
                     if not dry_run and del_ids:
                         with self._conn_ctx() as conn:
@@ -469,9 +483,18 @@ class MySQLDeduplicator:
                 temp_table = self._make_temp_table_name(table)
                 drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
                 create_temp_where = f"WHERE `{time_col}` = '{date_val}'"
+                # 判断分组字段是否有“更新时间”
+                has_update_time = any(col == '更新时间' for col in use_columns)
+                if has_update_time:
+                    keep_field = '更新时间'
+                    keep_func = 'MAX'
+                else:
+                    keep_field = pk_real
+                    keep_func = 'MAX' if self.keep_order == 'max' else 'MIN'
+                keep_alias = 'keep_val'
                 create_temp_sql = f"""
                 CREATE TABLE `{database}`.`{temp_table}` AS
-                SELECT MIN(`{pk_real}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
+                SELECT {keep_func}(`{keep_field}`) as `{keep_alias}`, {column_list}, COUNT(*) as `dup_count`
                 FROM `{database}`.`{table}`
                 {create_temp_where}
                 GROUP BY {column_list}
@@ -494,7 +517,7 @@ class MySQLDeduplicator:
                             while True:
                                 where_clauses = []
                                 if self.duplicate_keep_mode == 'keep_one':
-                                    where_clauses.append(f"t.`{pk_real}` <> tmp.`min_id`")
+                                    where_clauses.append(f"t.`{keep_field}` <> tmp.`{keep_alias}`")
                                 if where_sql.strip():
                                     where_clauses.append(where_sql.strip())
                                 where_full = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
@@ -752,7 +775,7 @@ class MySQLDeduplicator:
                             logger.warning('分区处理失败', {"库": database, "表": table, "日期": date_val, "异常": err, "func": sys._getframe().f_code.co_name})
                         total_dup += dup_count
                         total_del += affected_rows
-                logger.debug('单表完成', {"库": database, "表": table, "结果[重复, 删除]": (total_dup, total_del), '日期范围': f"{start_date} - {end_date}"})
+                logger.info('单表完成', {"库": database, "表": table, "结果[重复, 删除]": (total_dup, total_del), '日期范围': f"{start_date} - {end_date}", "唯一列": columns})
                 # 自动重排id列（仅当有实际删除时且reorder_id为True）
                 if reorder_id and total_del > 0:
                     try:
@@ -761,11 +784,11 @@ class MySQLDeduplicator:
                     except Exception as e:
                         logger.error('自动重排id列异常', {"库": database, "表": table, "异常": str(e)})
                 if affected_rows > 0:
-                    logger.info('单表完成(仅显示有删除的结果)', {"库": database, "表": table, "重复组": total_dup, "实际删除": total_del})
+                    logger.info('单表完成(仅显示有删除的结果)', {"库": database, "表": table, "重复组": total_dup, "实际删除": total_del, "唯一列": columns})
                 return (total_dup, total_del)
             # 没有date_column，直接全表去重
             result = self._deduplicate_table(database, table, columns, dry_run, use_python_dedup, date_val=None)
-            logger.debug('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result, '日期范围': '全表'})
+            logger.info('单表完成', {"库": database, "表": table, "结果[重复, 删除]": result, '日期范围': '全表', "唯一列": columns})
             dup_count, affected_rows = result
             if reorder_id and affected_rows > 0:
                 try:
@@ -774,7 +797,7 @@ class MySQLDeduplicator:
                 except Exception as e:
                     logger.error('自动重排id列异常', {"库": database, "表": table, "异常": str(e)})
             if affected_rows > 0:
-                logger.info('单表完成(仅显示有删除的结果)', {"库": database, "表": table, "重复组": dup_count, "实际删除": affected_rows})
+                logger.info('单表完成(仅显示有删除的结果)', {"库": database, "表": table, "重复组": dup_count, "实际删除": affected_rows, "唯一列": columns})
             return result
         except Exception as e:
             logger.error('发生全局错误', {"库": database, "表": table, 'func': sys._getframe().f_code.co_name, "发生全局错误": str(e)})
@@ -1314,18 +1337,24 @@ class MySQLDeduplicator:
 
 
 def main():
+    from mdbq.config import config
+    dir_path = os.path.expanduser("~")
+    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
+    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    # host = 'localhost'
+    
     deduplicator = MySQLDeduplicator(
-        username='root',
-        password='pwd',
-        host='localhost',
-        port=3306,
+        username=username,
+        password=password,
+        host=host,
+        port=port,
         max_workers= 2,
         batch_size=1000,
         skip_system_dbs=True,
         max_retries=3,
         retry_waiting_time=5,
         pool_size=30,
-        recent_month=1,
+        # recent_month=1,
         # date_range=['2025-06-09', '2025-06-10'],
         exclude_columns=['更新时间'],
         # exclude_databases=['测试库4'],
@@ -1338,6 +1367,7 @@ def main():
         #         "商品排行_2025",
         #     ],
         # },
+        keep_order='MAX',  # 保留重复组中指定列的最大值
     )
 
     # 全库去重(单线程)
@@ -1347,7 +1377,13 @@ def main():
     # deduplicator.deduplicate_database('数据引擎2', dry_run=False, parallel=True, reorder_id=True)
 
     # # 指定表去重(使用特定列)
-    deduplicator.deduplicate_table('安全组', '腾讯云cvm规则', columns=['平台', '本地主机', '端口范围', '授权ip'], dry_run=False, reorder_id=True)
+    deduplicator.deduplicate_table(
+        '京东数据3', 
+        '推广数据_京准通_2024', 
+        columns=['日期', '店铺名称', '产品线', '触发sku_id', '跟单sku_id', 'spu_id', '花费', '展现数', '点击数'], 
+        dry_run=False, 
+        reorder_id=True,
+        )
 
     # # 重排id列
     # deduplicator.reorder_id_column('my_db', 'my_table', 'id', dry_run=False, auto_drop_backup=True)

mdbq/mysql/uploader.py

@@ -323,7 +323,7 @@ class MySQLUploader:
             logger.error('无效的标识符', {'标识符': identifier})
             raise ValueError(f"无效的标识符: `{identifier}`")
         # 始终做特殊字符清理
-        cleaned = re.sub(r'[^-\uFFFF\w\u4e00-\u9fff$]', '_', identifier)
+        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '_', identifier)
         cleaned = re.sub(r'_+', '_', cleaned).strip('_')
         if not cleaned:
             logger.error('无法清理异常标识符', {'原始标识符': identifier})
@@ -332,6 +332,8 @@ class MySQLUploader:
             'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
             'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
         }
+        if len(cleaned) > 64:
+            cleaned = cleaned[:64]
         if cleaned.lower() in mysql_keywords:
             logger.debug('存在MySQL保留字', {'标识符': cleaned})
             return f"`{cleaned}`"
@@ -423,11 +425,11 @@ class MySQLUploader:
         # UNIQUE KEY定义
         unique_defs = []
         if unique_keys:
-            for idx, unique_cols in enumerate(unique_keys):
+            for unique_cols in unique_keys:
                 if not unique_cols:
                     continue
                 safe_unique_cols = [self._normalize_col(col) for col in unique_cols]
-                unique_name = f"uniq_{'_'.join(safe_unique_cols)}_{idx}"
+                unique_name = f"uniq_{'_'.join(safe_unique_cols)}"
                 unique_defs.append(f"UNIQUE KEY `{unique_name}` (`{'`,`'.join(safe_unique_cols)}`)")
         index_defs = list(set(index_defs))
         all_defs = column_defs + [primary_key_sql] + index_defs + unique_defs
@@ -651,7 +653,7 @@ class MySQLUploader:
         添加UNIQUE KEY
         """
         safe_cols = [self._normalize_col(col) for col in unique_cols]
-        unique_name = f"uniq_{'_'.join(safe_cols)}_{int(time.time()*1000)%100000}"
+        unique_name = f"uniq_{'_'.join(safe_cols)}"
         sql = f'ALTER TABLE `{db_name}`.`{table_name}` ADD UNIQUE KEY `{unique_name}` ({','.join(f'`{col}`' for col in safe_cols)})'
         try:
             with self._get_connection() as conn:

{mdbq-3.12.3.dist-info → mdbq-3.12.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.12.3
+Version: 3.12.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.12.3.dist-info → mdbq-3.12.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=9x3pJeqVdjIfn83Ln01YuTBiHKPCTeK3xkXQT4NcYno,18
+mdbq/__version__.py,sha256=V3m3JOLEsFbctTVRE9dNc1iuNQahT_FbWxcHtAoWWqc,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -8,10 +8,10 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=Crw6LwVo3I3IUbzIETu8f46Quza3CTCh-qYf4edbBPo,24139
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=uBRM2cBF-gzkFFrmBSKqBd_LLO-K67LYUJqpF9Fs928,70561
+mdbq/mysql/deduplicator.py,sha256=MYcNe0rwWOFS2Bqac8yGwwothlz8H--cOi6yuZf3qIs,72602
 mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
-mdbq/mysql/uploader.py,sha256=szX6t4SObBF6fbHT2s5ixfh1-c288cigsJ66pFE02Qg,70266
+mdbq/mysql/uploader.py,sha256=ekpPaJypnuwxi2v42e-khqwT_eZ5LRl1ylQP492xbkk,70271
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -24,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
-mdbq-3.12.3.dist-info/METADATA,sha256=kkujbmKfbA4mOHCloHBALjK3jI2sNoft1yyHmpbnmoI,364
-mdbq-3.12.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.12.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.12.3.dist-info/RECORD,,
+mdbq-3.12.4.dist-info/METADATA,sha256=bptLkLuByUNJJnZ_ruWRjxeG2LOFvaqK26M3MtQkf78,364
+mdbq-3.12.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.12.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.12.4.dist-info/RECORD,,