mdbq 3.10.7__py3-none-any.whl → 3.10.9__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.10.7'
+VERSION = '3.10.9'

mdbq/aggregation/optimize.py

@@ -457,6 +457,7 @@ def main():
         '安全组',
         # '视频数据',
         # '聚合数据',
+        '数据引擎2'
     ]
     tables_list = op.get_table_in_databases(db_list=db_list, reset_id=False)
     op.deduplicate(

mdbq/aggregation/query_data.py

@@ -3995,6 +3995,7 @@ def main(days=150, months=3):
         "推广数据2",
         "推广数据_淘宝店",
         "推广数据_奥莱店",
+        "推广数据_圣积天猫店",
         "爱库存2",
         "生意参谋3",
         "生意经3",
@@ -4003,6 +4004,7 @@ def main(days=150, months=3):
         '商品人群画像2',
         '市场数据3',
         '回传数据',
+        '数据引擎2',
     ]
     # 使用 ThreadPoolExecutor 来并行运行
     # with concurrent.futures.ThreadPoolExecutor() as executor:

mdbq/log/mylogger.py

@@ -239,8 +239,9 @@ class MyLogger:
                         log_data['异常'] = self.formatException(record.exc_info)
 
                     # 过滤敏感信息
-                    if hasattr(record, '过滤'):
-                        for field in record.sensitive_fields:
+                    if hasattr(record, 'extra_data') and '过滤' in record.extra_data:
+                        sensitive_fields = record.extra_data['过滤']
+                        for field in sensitive_fields:
                             if field in log_data:
                                 log_data[field] = '***'
                             if isinstance(log_data.get('message'), str):
@@ -447,8 +448,11 @@ class MyLogger:
             try:
                 self._log_queue.put((level, message, extra), timeout=0.1)
             except queue.Full:
-                # 队列满时降级为同步日志
-                self._sync_log(level, f"[ASYNC QUEUE FULL] {message}", extra)
+                # 队列满时降级为同步日志，添加队列状态信息到extra
+                if extra is None:
+                    extra = {}
+                extra['queue_status'] = 'full'
+                self._sync_log(level, message, extra)
         else:
             self._sync_log(level, message, extra)
 
@@ -509,21 +513,6 @@ class MyLogger:
         if not extra:
             extra = {}
 
-        # # 获取完整的异常堆栈
-        # tb = exc_info.__traceback__
-        # while tb.tb_next:
-        #     tb = tb.tb_next  # 获取最内层的堆栈帧
-        #
-        # extra.update({
-        #     'module': tb.tb_frame.f_globals.get('__name__', ''),
-        #     'function': tb.tb_frame.f_code.co_name,
-        #     'line': tb.tb_lineno,
-        #     'file': tb.tb_frame.f_code.co_filename,
-        #     '异常': str(exc_info),
-        #     '类型': exc_info.__class__.__name__,
-        #     '堆栈': self._format_traceback(exc_info)
-        # })
-
         # 使用inspect获取调用栈
         frame = inspect.currentframe()
         try:

mdbq/mysql/deduplicator.py

@@ -24,7 +24,7 @@ logger = mylogger.MyLogger(
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样50%的DEBUG/INFO日志
+    sample_rate=1,  # 采样DEBUG/INFO日志, 0.5表示50%的日志会被采样
     sensitive_fields=[],  #  敏感字段列表
 )
 
@@ -72,8 +72,9 @@ class MySQLDeduplicator:
             skip_system_dbs: bool = True,
             max_retries: int = 3,
             retry_interval: int = 5,
-            pool_size: int = 5
-    ):
+            pool_size: int = 5,
+            primary_key: str = 'id'
+    ) -> None:
         """
         初始化去重处理器
 
@@ -88,6 +89,7 @@ class MySQLDeduplicator:
         :param max_retries: 最大重试次数
         :param retry_interval: 重试间隔(秒)
         :param pool_size: 连接池大小
+        :param primary_key: 主键列名，默认为'id'
         """
         # 连接池状态标志
         self._closed = False
@@ -110,6 +112,7 @@ class MySQLDeduplicator:
         self.skip_system_dbs = skip_system_dbs
         self.max_retries = max_retries
         self.retry_interval = retry_interval
+        self.primary_key = primary_key
 
         # 线程安全控制
         self._lock = threading.Lock()
@@ -118,7 +121,7 @@ class MySQLDeduplicator:
         # 系统数据库列表
         self.SYSTEM_DATABASES = {'information_schema', 'mysql', 'performance_schema', 'sys'}
 
-    def _get_connection(self):
+    def _get_connection(self) -> pymysql.connections.Connection:
         """从连接池获取连接"""
         if self._closed:
             raise ConnectionError("连接池已关闭")
@@ -131,7 +134,7 @@ class MySQLDeduplicator:
             raise ConnectionError(f"连接数据库失败: {str(e)}")
 
     @staticmethod
-    def _retry_on_failure(func):
+    def _retry_on_failure(func: Any) -> Any:
         """重试装饰器"""
 
         @wraps(func)
@@ -187,7 +190,7 @@ class MySQLDeduplicator:
 
     @_retry_on_failure
     def _get_table_columns(self, database: str, table: str) -> List[str]:
-        """获取表的列名(排除id列)"""
+        """获取表的列名(排除主键列)"""
         sql = """
         SELECT COLUMN_NAME 
         FROM INFORMATION_SCHEMA.COLUMNS 
@@ -199,7 +202,7 @@ class MySQLDeduplicator:
             with conn.cursor() as cursor:
                 cursor.execute(sql, (database, table))
                 return [row['COLUMN_NAME'] for row in cursor.fetchall()
-                        if row['COLUMN_NAME'].lower() != 'id']
+                        if row['COLUMN_NAME'].lower() != self.primary_key.lower()]
 
     def _acquire_table_lock(self, database: str, table: str) -> bool:
         """获取表处理锁，防止并发处理同一张表"""
@@ -212,7 +215,7 @@ class MySQLDeduplicator:
             self._processing_tables.add(key)
             return True
 
-    def _release_table_lock(self, database: str, table: str):
+    def _release_table_lock(self, database: str, table: str) -> None:
         """释放表处理锁"""
         key = f"{database}.{table}"
 
@@ -255,7 +258,7 @@ class MySQLDeduplicator:
             if invalid_columns:
                 logger.warning(
                     f"表 {database}.{table} 中不存在以下列: {invalid_columns}，使用有效列",
-                    {'invalid_columns': invalid_columns}
+                    {'invalid_columns': list(invalid_columns)}
                 )
                 use_columns = [col for col in use_columns if col in all_columns]
 
@@ -265,14 +268,17 @@ class MySQLDeduplicator:
 
             # 构建去重SQL
             column_list = ', '.join([f'`{col}`' for col in use_columns])
-            # temp_table = f"temp_{table}_{int(time.time())}"
-            temp_table = f"temp_{table}_dedup_{os.getpid()}"  # 使用进程ID构建临时表
-            temp_table = re.sub(r'[^a-zA-Z0-9_]', '_', temp_table)  # 确保表名合法
-
-            # 使用临时表方案处理去重，避免锁表问题
+            # 临时表名限制64字符以内
+            temp_table = f"temp_{table}_dedup_{os.getpid()}"
+            temp_table = re.sub(r'[^a-zA-Z0-9_]', '_', temp_table)[:60]
+            pk = self.primary_key
+            # 校验主键列是否存在
+            if pk not in all_columns and pk != 'id':
+                logger.error(f"表 {database}.{table} 不存在主键列 {pk}")
+                return (0, 0)
             create_temp_sql = f"""
             CREATE TABLE `{database}`.`{temp_table}` AS
-            SELECT MIN(`id`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
+            SELECT MIN(`{pk}`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
             FROM `{database}`.`{table}`
             GROUP BY {column_list}
             HAVING COUNT(*) > 1
@@ -280,7 +286,7 @@ class MySQLDeduplicator:
 
             delete_dup_sql = f"""
             DELETE FROM `{database}`.`{table}`
-            WHERE `id` NOT IN (
+            WHERE `{pk}` NOT IN (
                 SELECT `min_id` FROM `{database}`.`{temp_table}`
             ) AND ({' OR '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
             """
@@ -292,7 +298,8 @@ class MySQLDeduplicator:
                     # 创建临时表统计重复数据
                     cursor.execute(create_temp_sql)
                     cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`")
-                    dup_count = cursor.fetchone()['cnt']
+                    dup_count_row = cursor.fetchone()
+                    dup_count = dup_count_row['cnt'] if dup_count_row and 'cnt' in dup_count_row else 0
 
                     if dup_count == 0:
                         logger.info(f"表 {database}.{table} 没有重复数据")
@@ -557,7 +564,7 @@ class MySQLDeduplicator:
                 cursor.execute(sql, (database, table))
                 return bool(cursor.fetchone())
 
-    def close(self):
+    def close(self) -> None:
         """关闭连接池"""
         try:
             if hasattr(self, 'pool') and self.pool and not self._closed:
@@ -567,17 +574,17 @@ class MySQLDeduplicator:
         except Exception as e:
             logger.error(f"关闭连接池时出错: {str(e)}", {'error_type': type(e).__name__})
 
-    def __enter__(self):
+    def __enter__(self) -> 'MySQLDeduplicator':
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(self, exc_type: Optional[type], exc_val: Optional[BaseException], exc_tb: Optional[Any]) -> None:
         self.close()
 
 
 def main():
     deduplicator = MySQLDeduplicator(
         username='root',
-        password='188988yang188',
+        password='pwd',
         host='localhost',
         port=3306
     )
@@ -595,4 +602,5 @@ def main():
     deduplicator.close()
 
 if __name__ == '__main__':
-    main()
+    # main()
+    pass