mdbq 4.2.0__py3-none-any.whl → 4.2.2__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.2.0'
+VERSION = '4.2.2'

mdbq/mysql/uploader.py

@@ -1,20 +1,24 @@
 # -*- coding:utf-8 -*-
 """
-MySQL数据上传器 - 重构版本
-提供高可用、易维护的MySQL数据上传功能
+MySQL数据上传
 """
-
 import datetime
 import time
 import json
 import re
-from typing import Union, List, Dict, Optional, Any, Tuple
+import io
+from typing import Union, List, Dict, Optional, Any, Tuple, Iterator
 from functools import wraps
 from decimal import Decimal, InvalidOperation
 import math
-
+import concurrent.futures
+import threading
+from queue import Queue
 import pymysql
 import pandas as pd
+import psutil
+import enum
+import ipaddress
 from dbutils.pooled_db import PooledDB
 from mdbq.log import mylogger
 # from mdbq.myconf import myconf
@@ -87,12 +91,35 @@ class DatabaseConnectionManager:
 class DataTypeInferrer:
     """数据类型推断器"""
     
+    # 自定义类型映射注册表
+    _custom_type_handlers = {}
+    
+    @classmethod
+    def register_type_handler(cls, type_name: str, handler_func):
+        """
+        注册自定义类型处理器
+        
+        :param type_name: 类型名称
+        :param handler_func: 处理函数，接收value参数，返回MySQL类型字符串或None
+        """
+        cls._custom_type_handlers[type_name] = handler_func
+    
     @staticmethod
     def infer_mysql_type(value: Any) -> str:
         """推断MySQL数据类型"""
         if value is None or str(value).lower() in ['', 'none', 'nan']:
             return 'VARCHAR(255)'
         
+        # 检查自定义类型处理器
+        for type_name, handler in DataTypeInferrer._custom_type_handlers.items():
+            try:
+                result = handler(value)
+                if result:
+                    return result
+            except Exception:
+                continue
+        
+        # Python基本类型
         if isinstance(value, bool):
             return 'TINYINT(1)'
         elif isinstance(value, int):
@@ -109,42 +136,231 @@ class DataTypeInferrer:
         elif isinstance(value, (list, dict)):
             return 'JSON'
         elif isinstance(value, str):
-            # 尝试判断是否是日期时间
-            if DataValidator.is_datetime_string(value):
-                return 'DATETIME'
-            
-            # 根据字符串长度选择类型
-            length = len(value)
-            if length <= 255:
-                return 'VARCHAR(255)'
-            elif length <= 65535:
-                return 'TEXT'
-            else:
-                return 'LONGTEXT'
+            return DataTypeInferrer._infer_string_type(value)
+        
+        # 处理枚举类型
+        if hasattr(value, '__class__') and hasattr(value.__class__, '__bases__'):
+            # 检查是否是枚举类型
+            if isinstance(value, enum.Enum):
+                # 根据枚举值的类型决定MySQL类型
+                enum_value = value.value
+                if isinstance(enum_value, int):
+                    return 'INT'
+                elif isinstance(enum_value, str):
+                    max_len = max(len(str(item.value)) for item in value.__class__)
+                    return f'VARCHAR({min(max_len * 2, 255)})'
+                else:
+                    return 'VARCHAR(255)'
+        
+        # 处理其他特殊类型
+        value_str = str(value)
+        
+        # UUID检测
+        if DataTypeInferrer._is_uuid(value_str):
+            return 'CHAR(36)'
+        
+        # IP地址检测
+        if DataTypeInferrer._is_ip_address(value_str):
+            return 'VARCHAR(45)'  # 支持IPv6
+        
+        # 邮箱检测
+        if DataTypeInferrer._is_email(value_str):
+            return 'VARCHAR(255)'
+        
+        # URL检测
+        if DataTypeInferrer._is_url(value_str):
+            return 'TEXT'
         
-        return 'VARCHAR(255)'
+        # 默认字符串处理
+        return DataTypeInferrer._infer_string_type(value_str)
+    
+    @staticmethod
+    def _infer_string_type(value: str) -> str:
+        """推断字符串类型"""
+        # 尝试判断是否是日期时间
+        if DataValidator.is_datetime_string(value):
+            return 'DATETIME'
+        
+        # 数值字符串检测
+        if DataTypeInferrer._is_numeric_string(value):
+            if '.' in value or 'e' in value.lower():
+                return 'DECIMAL(20,6)'
+            else:
+                try:
+                    int_val = int(value)
+                    if -2147483648 <= int_val <= 2147483647:
+                        return 'INT'
+                    else:
+                        return 'BIGINT'
+                except ValueError:
+                    pass
+        
+        # 根据字符串长度选择类型
+        length = len(value)
+        if length <= 255:
+            return 'VARCHAR(255)'
+        elif length <= 65535:
+            return 'TEXT'
+        else:
+            return 'LONGTEXT'
     
     @staticmethod
-    def infer_types_from_data(data: List[Dict]) -> Dict[str, str]:
-        """从数据中推断所有列的类型"""
+    def _is_uuid(value: str) -> bool:
+        """检测是否是UUID格式"""
+        uuid_pattern = r'^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'
+        return bool(re.match(uuid_pattern, value.lower()))
+    
+    @staticmethod
+    def _is_ip_address(value: str) -> bool:
+        """检测是否是IP地址"""
+        try:
+            ipaddress.ip_address(value)
+            return True
+        except ValueError:
+            return False
+    
+    @staticmethod
+    def _is_email(value: str) -> bool:
+        """检测是否是邮箱地址"""
+        email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+        return bool(re.match(email_pattern, value))
+    
+    @staticmethod
+    def _is_url(value: str) -> bool:
+        """检测是否是URL"""
+        url_pattern = r'^https?://[^\s/$.?#].[^\s]*$'
+        return bool(re.match(url_pattern, value, re.IGNORECASE))
+    
+    @staticmethod
+    def _is_numeric_string(value: str) -> bool:
+        """检测是否是数值字符串"""
+        try:
+            float(value)
+            return True
+        except ValueError:
+            return False
+    
+    @staticmethod
+    def infer_types_from_data(data: List[Dict], sample_size: int = 100) -> Dict[str, str]:
+        """
+        从数据中推断所有列的类型
+        
+        :param data: 数据列表
+        :param sample_size: 采样大小，避免检查过多数据
+        """
         if not data:
             return {}
         
         type_map = {}
-        for row in data[:10]:  # 只检查前10行
+        type_candidates = {}  # 存储每列的候选类型
+        
+        # 采样数据进行类型推断
+        sample_data = data[:sample_size] if len(data) > sample_size else data
+        
+        for row in sample_data:
             for col, value in row.items():
                 # 跳过系统列
                 if col.lower() in ['id', 'create_at', 'update_at']:
                     continue
-                if col not in type_map and value is not None:
-                    type_map[col] = DataTypeInferrer.infer_mysql_type(value)
+                
+                if value is not None and str(value).strip():
+                    mysql_type = DataTypeInferrer.infer_mysql_type(value)
+                    
+                    if col not in type_candidates:
+                        type_candidates[col] = []
+                    type_candidates[col].append(mysql_type)
+        
+        # 为每列选择最合适的类型
+        for col, types in type_candidates.items():
+            type_map[col] = DataTypeInferrer._select_best_type(types)
         
-        # 自动添加系统列类型定义
+        # 自动添加系统列类型定义（id列只在新建表时添加）
         type_map['id'] = 'BIGINT'
         type_map['create_at'] = 'TIMESTAMP'
         type_map['update_at'] = 'TIMESTAMP'
         
         return type_map
+    
+    @staticmethod
+    def _select_best_type(type_candidates: List[str]) -> str:
+        """
+        从候选类型中选择最佳类型
+        
+        优先级：JSON > LONGTEXT > TEXT > VARCHAR > DECIMAL > BIGINT > INT > DATETIME > DATE
+        """
+        if not type_candidates:
+            return 'VARCHAR(255)'
+        
+        # 类型优先级映射
+        type_priority = {
+            'JSON': 10,
+            'LONGTEXT': 9,
+            'TEXT': 8,
+            'VARCHAR': 7,
+            'DECIMAL': 6,
+            'BIGINT': 5,
+            'INT': 4,
+            'DATETIME': 3,
+            'DATE': 2,
+            'TINYINT': 1
+        }
+        
+        # 找到优先级最高的类型
+        best_type = 'VARCHAR(255)'
+        best_priority = 0
+        
+        for candidate in set(type_candidates):
+            # 提取基础类型名
+            base_type = candidate.split('(')[0].upper()
+            priority = type_priority.get(base_type, 0)
+            
+            if priority > best_priority:
+                best_priority = priority
+                best_type = candidate
+        
+        return best_type
+
+
+# 注册一些常用的自定义类型处理器
+def register_common_type_handlers():
+    """注册常用的自定义类型处理器"""
+    
+    def handle_phone_number(value):
+        """处理电话号码"""
+        if isinstance(value, str):
+            # 中国手机号码格式
+            if re.match(r'^1[3-9]\d{9}$', value):
+                return 'VARCHAR(11)'
+            # 国际电话号码格式
+            if re.match(r'^\+?[1-9]\d{1,14}$', value):
+                return 'VARCHAR(20)'
+        return None
+    
+    def handle_id_card(value):
+        """处理身份证号"""
+        if isinstance(value, str):
+            # 中国身份证号码
+            if re.match(r'^\d{17}[\dXx]$', value):
+                return 'CHAR(18)'
+        return None
+    
+    def handle_json_string(value):
+        """处理JSON字符串"""
+        if isinstance(value, str):
+            try:
+                json.loads(value)
+                return 'JSON'
+            except (ValueError, TypeError):
+                pass
+        return None
+    
+    # 注册处理器
+    DataTypeInferrer.register_type_handler('phone', handle_phone_number)
+    DataTypeInferrer.register_type_handler('id_card', handle_id_card)
+    DataTypeInferrer.register_type_handler('json_string', handle_json_string)
+
+# 自动注册常用类型处理器
+register_common_type_handlers()
 
 
 class DataValidator:
@@ -376,6 +592,80 @@ class TableManager:
                 )
                 return bool(cursor.fetchone())
     
+    def get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
+        """获取表的列信息"""
+        db_name = self._sanitize_identifier(db_name)
+        table_name = self._sanitize_identifier(table_name)
+        
+        with self.conn_mgr.get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute("""
+                    SELECT COLUMN_NAME, COLUMN_TYPE 
+                    FROM INFORMATION_SCHEMA.COLUMNS 
+                    WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
+                """, (db_name, table_name))
+                
+                columns = {}
+                for row in cursor.fetchall():
+                    columns[row['COLUMN_NAME']] = row['COLUMN_TYPE']
+                return columns
+    
+    def get_table_primary_key(self, db_name: str, table_name: str) -> Optional[str]:
+        """获取表的主键列名"""
+        db_name = self._sanitize_identifier(db_name)
+        table_name = self._sanitize_identifier(table_name)
+        
+        with self.conn_mgr.get_connection() as conn:
+            with conn.cursor() as cursor:
+                cursor.execute("""
+                    SELECT COLUMN_NAME 
+                    FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE 
+                    WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s 
+                    AND CONSTRAINT_NAME = 'PRIMARY'
+                """, (db_name, table_name))
+                
+                result = cursor.fetchone()
+                return result['COLUMN_NAME'] if result else None
+    
+    def ensure_system_columns(self, db_name: str, table_name: str):
+        """确保表有系统列，如果没有则添加（保持原有主键结构）"""
+        existing_columns = self.get_table_columns(db_name, table_name)
+        existing_primary_key = self.get_table_primary_key(db_name, table_name)
+        
+        with self.conn_mgr.get_connection() as conn:
+            with conn.cursor() as cursor:
+                # 只有在表没有主键且没有id列时，才添加id主键
+                if existing_primary_key is None and 'id' not in existing_columns:
+                    cursor.execute(f"""
+                        ALTER TABLE `{db_name}`.`{table_name}` 
+                        ADD COLUMN `id` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST
+                    """)
+                    logger.info('自动添加id主键列', {'database': db_name, 'table': table_name})
+                elif existing_primary_key is not None:
+                    logger.debug('表已有主键，保持原有结构', {
+                        'database': db_name, 
+                        'table': table_name, 
+                        'primary_key': existing_primary_key
+                    })
+                
+                # 检查并添加create_at列
+                if 'create_at' not in existing_columns:
+                    cursor.execute(f"""
+                        ALTER TABLE `{db_name}`.`{table_name}` 
+                        ADD COLUMN `create_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+                    """)
+                    logger.info('自动添加create_at列', {'database': db_name, 'table': table_name})
+                
+                # 检查并添加update_at列
+                if 'update_at' not in existing_columns:
+                    cursor.execute(f"""
+                        ALTER TABLE `{db_name}`.`{table_name}` 
+                        ADD COLUMN `update_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+                    """)
+                    logger.info('自动添加update_at列', {'database': db_name, 'table': table_name})
+                
+                conn.commit()
+    
     def create_table(self, db_name: str, table_name: str, columns: Dict[str, str], 
                     primary_keys: Optional[List[str]] = None, 
                     unique_keys: Optional[List[List[str]]] = None):
@@ -479,24 +769,49 @@ class DataProcessor:
     """数据处理器"""
     
     @staticmethod
-    def normalize_data(data: Union[Dict, List[Dict], pd.DataFrame]) -> List[Dict]:
-        """标准化数据格式为字典列表"""
+    def normalize_data(data: Union[Dict, List[Dict], pd.DataFrame], 
+                      chunk_size: int = 5000, 
+                      memory_limit_mb: int = 100) -> Iterator[List[Dict]]:
+        """
+        标准化数据格式为分块迭代器
+        
+        :param data: 输入数据
+        :param chunk_size: 每个chunk的大小
+        :param memory_limit_mb: 内存限制(MB)，超过时自动调整chunk_size
+        """
+        # 动态调整chunk_size基于可用内存
+        available_memory_mb = psutil.virtual_memory().available / 1024 / 1024
+        if available_memory_mb < memory_limit_mb * 2:
+            chunk_size = min(chunk_size, 1000)  # 内存紧张时减小chunk
+        
         if isinstance(data, pd.DataFrame):
-            return data.to_dict('records')
+            # 对于大DataFrame，使用更高效的分块方式
+            if len(data) > 50000:
+                # 大数据集使用pandas的分块读取
+                for chunk in pd.read_csv(io.StringIO(data.to_csv(index=False)), chunksize=chunk_size):
+                    yield chunk.to_dict('records')
+            else:
+                for i in range(0, len(data), chunk_size):
+                    chunk = data.iloc[i:i + chunk_size]
+                    yield chunk.to_dict('records')
         elif isinstance(data, dict):
-            return [data]
-        elif isinstance(data, list) and all(isinstance(item, dict) for item in data):
-            return data
+            yield [data]
+        elif isinstance(data, list):
+            if all(isinstance(item, dict) for item in data):
+                for i in range(0, len(data), chunk_size):
+                    yield data[i:i + chunk_size]
+            else:
+                raise ValueError("列表中必须全部是字典")
         else:
             raise ValueError("数据格式必须是字典、字典列表或DataFrame")
     
     @staticmethod
-    def prepare_data_for_insert(data: List[Dict], set_typ: Dict[str, str], 
+    def prepare_data_for_insert(data_chunk: List[Dict], set_typ: Dict[str, str], 
                                allow_null: bool = False) -> List[Dict]:
         """准备插入数据"""
         prepared_data = []
         
-        for row_idx, row in enumerate(data, 1):
+        for row_idx, row in enumerate(data_chunk, 1):
             prepared_row = {}
             
             for col_name, col_type in set_typ.items():
@@ -523,13 +838,13 @@ class DataProcessor:
         return prepared_data
     
     @staticmethod
-    def partition_data_by_date(data: List[Dict], date_column: str, 
+    def partition_data_by_date(data_chunk: List[Dict], date_column: str, 
                               partition_by: str) -> Dict[str, List[Dict]]:
-        """按日期分区数据"""
+        """按日期分区数据块"""
         partitioned = {}
         table_manager = TableManager(None, None)  # 只用静态方法
         
-        for row in data:
+        for row in data_chunk:
             if date_column not in row:
                 logger.warning('缺少分区日期列', {'列名': date_column, '行数据': row})
                 continue
@@ -583,34 +898,59 @@ class DataInserter:
     def _execute_batch_insert(self, sql: str, data: List[Dict], 
                              columns: List[str]) -> Tuple[int, int, int]:
         """执行批量插入"""
-        batch_size = min(1000, len(data))
+        # 动态调整批次大小
+        estimated_row_size = len(str(data[0])) if data else 100
+        max_packet_size = 16 * 1024 * 1024  # 16MB MySQL默认限制
+        optimal_batch_size = min(
+            max_packet_size // (estimated_row_size * len(columns)),
+            2000,  # 最大批次
+            len(data)
+        )
+        batch_size = max(100, optimal_batch_size)  # 最小100条
+        
         total_inserted = 0
         total_skipped = 0
         total_failed = 0
         
         with self.conn_mgr.get_connection() as conn:
             with conn.cursor() as cursor:
-                for i in range(0, len(data), batch_size):
-                    batch = data[i:i + batch_size]
-                    values_list = []
-                    
-                    for row in batch:
-                        values = [self._ensure_basic_type(row.get(col)) for col in columns]
-                        values_list.append(values)
+                # 预处理所有数据，减少循环中的处理开销
+                all_values = []
+                for row in data:
+                    values = [self._ensure_basic_type(row.get(col)) for col in columns]
+                    all_values.append(values)
+                
+                # 分批处理，使用更大的事务批次
+                transaction_size = min(5000, len(all_values))  # 每个事务处理的记录数
+                
+                for tx_start in range(0, len(all_values), transaction_size):
+                    tx_end = min(tx_start + transaction_size, len(all_values))
+                    tx_values = all_values[tx_start:tx_end]
                     
                     try:
-                        cursor.executemany(sql, values_list)
+                        # 开始事务
+                        conn.begin()
+                        
+                        # 在事务内分批执行，成功后直接累加
+                        for i in range(0, len(tx_values), batch_size):
+                            batch_values = tx_values[i:i + batch_size]
+                            
+                            try:
+                                cursor.executemany(sql, batch_values)
+                                total_inserted += len(batch_values)
+                            except pymysql.err.IntegrityError:
+                                total_skipped += len(batch_values)
+                                logger.debug('批量插入唯一约束冲突，跳过', {'批次大小': len(batch_values)})
+                            except Exception as e:
+                                logger.error('批量插入失败', {'错误': str(e), '批次大小': len(batch_values)})
+                                raise
+                        
                         conn.commit()
-                        affected = cursor.rowcount if cursor.rowcount is not None else len(batch)
-                        total_inserted += affected
-                    except pymysql.err.IntegrityError:
-                        conn.rollback()
-                        total_skipped += len(batch)
-                        logger.debug('批量插入唯一约束冲突，跳过', {'批次大小': len(batch)})
+                        
                     except Exception as e:
                         conn.rollback()
-                        total_failed += len(batch)
-                        logger.error('批量插入失败', {'错误': str(e), '批次大小': len(batch)})
+                        logger.error('事务执行失败，已回滚', {'错误': str(e)})
+                        total_failed += len(tx_values)
         
         return total_inserted, total_skipped, total_failed
     
@@ -655,13 +995,13 @@ def retry_on_failure(max_retries: int = 3, delay: int = 1):
 
 class MySQLUploader:
     """
-    MySQL数据上传器 - 重构版本
+    MySQL数据上传器
     
     特性：
     - 自动为每个表添加id（BIGINT自增主键）、create_at、update_at时间戳列
     - 支持自动建表、分表、数据类型推断
     - 高可用连接池管理和重试机制
-    - 批量插入优化
+    - 流式批量插入优化
     """
     
     def __init__(self, username: str, password: str, host: str = 'localhost', 
@@ -705,7 +1045,7 @@ class MySQLUploader:
                    partition_by: Optional[str] = None,
                    partition_date_column: str = '日期',
                    update_on_duplicate: bool = False,
-                   unique_keys: Optional[List[List[str]]] = None) -> bool:
+                   unique_keys: Optional[List[List[str]]] = None) -> Dict[str, Any]:
         """
         上传数据到MySQL数据库
         
@@ -714,120 +1054,256 @@ class MySQLUploader:
         - create_at: 创建时间戳（插入时自动设置）
         - update_at: 更新时间戳（插入和更新时自动设置）
         
-        :param db_name: 数据库名（会自动转为小写）
-        :param table_name: 表名（会自动转为小写）
-        :param data: 要上传的数据
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要上传的数据，支持字典、字典列表、DataFrame
         :param set_typ: 列类型定义，如果为None则自动推断（无需包含系统列）
         :param allow_null: 是否允许空值
         :param partition_by: 分表方式('year'或'month')
         :param partition_date_column: 分表日期列名
         :param update_on_duplicate: 遇到重复数据时是否更新
         :param unique_keys: 唯一约束列表（无需包含系统列）
-        :return: 上传是否成功
+        :return: 上传结果详情
         """
         db_name = db_name.lower()
         table_name = table_name.lower()
+        
+        result = {
+            'success': False,
+            'inserted_rows': 0,
+            'skipped_rows': 0,
+            'failed_rows': 0,
+            'tables_created': []
+        }
+        
         try:
-            start_time = time.time()
+            # 计算原始数据大小
+            original_data_size = 0
+            if isinstance(data, (pd.DataFrame, list)):
+                original_data_size = len(data)
+            elif isinstance(data, dict):
+                original_data_size = 1
             
-            # 标准化数据
+            # 标准化数据为流式迭代器
             normalized_data = DataProcessor.normalize_data(data)
-            if not normalized_data:
-                logger.warning('数据为空，跳过上传')
-                return True
             
             # 推断或验证列类型
             if set_typ is None:
-                set_typ = DataTypeInferrer.infer_types_from_data(normalized_data)
-                logger.info('自动推断数据类型', {'类型映射': set_typ})
+                # 取第一个chunk进行类型推断
+                first_chunk = next(iter(normalized_data))
+                set_typ = DataTypeInferrer.infer_types_from_data(first_chunk)
+                # 重新创建迭代器
+                normalized_data = DataProcessor.normalize_data(data)
+                logger.debug('自动推断数据类型', {'类型映射': set_typ})
             
             # 确保数据库存在
             self.table_mgr.ensure_database_exists(db_name)
             
             # 处理分表逻辑
             if partition_by:
-                return self._handle_partitioned_upload(
+                upload_result = self._handle_partitioned_upload(
                     db_name, table_name, normalized_data, set_typ,
                     partition_by, partition_date_column, allow_null,
                     update_on_duplicate, unique_keys
                 )
             else:
-                return self._handle_single_table_upload(
+                upload_result = self._handle_single_table_upload(
                     db_name, table_name, normalized_data, set_typ,
                     allow_null, update_on_duplicate, unique_keys
                 )
-        
+            
+            # 合并结果
+            result.update(upload_result)
+            result['success'] = upload_result.get('failed_rows', 0) == 0
+            
         except Exception as e:
             logger.error('数据上传失败', {
                 '数据库': db_name,
                 '表名': table_name,
                 '错误': str(e)
             })
-            return False
+            result['success'] = False
+        
+        return result
     
     def _handle_single_table_upload(self, db_name: str, table_name: str,
-                                   data: List[Dict], set_typ: Dict[str, str],
+                                   data: Iterator[List[Dict]], 
+                                   set_typ: Dict[str, str],
                                    allow_null: bool, update_on_duplicate: bool,
-                                   unique_keys: Optional[List[List[str]]]) -> bool:
+                                   unique_keys: Optional[List[List[str]]]) -> Dict[str, Any]:
         """处理单表上传"""
+        result = {
+            'inserted_rows': 0,
+            'skipped_rows': 0,
+            'failed_rows': 0,
+            'tables_created': []
+        }
+        
         # 确保表存在
         if not self.table_mgr.table_exists(db_name, table_name):
             self.table_mgr.create_table(db_name, table_name, set_typ, 
                                        unique_keys=unique_keys)
+            result['tables_created'].append(f"{db_name}.{table_name}")
+        else:
+            # 表已存在，确保有时间戳列（但保持原有主键结构）
+            self.table_mgr.ensure_system_columns(db_name, table_name)
         
-        # 准备数据
-        prepared_data = DataProcessor.prepare_data_for_insert(
-            data, set_typ, allow_null
-        )
-        
-        # 插入数据
-        inserted, skipped, failed = self.data_inserter.insert_data(
-            db_name, table_name, prepared_data, set_typ, update_on_duplicate
-        )
+        # 流式处理每个数据块
+        for chunk in data:
+            if not chunk:
+                continue
+            
+            prepared_chunk = DataProcessor.prepare_data_for_insert(
+                chunk, set_typ, allow_null
+            )
+            
+            inserted, skipped, failed = self.data_inserter.insert_data(
+                db_name, table_name, prepared_chunk, set_typ, update_on_duplicate
+            )
+            
+            result['inserted_rows'] += inserted
+            result['skipped_rows'] += skipped
+            result['failed_rows'] += failed
         
         logger.info('单表上传完成', {
             '数据库': db_name,
             '表名': table_name,
-            '总数': len(data),
-            '插入': inserted,
-            '跳过': skipped,
-            '失败': failed
+            '插入': result['inserted_rows'],
+            '跳过': result['skipped_rows'],
+            '失败': result['failed_rows']
         })
         
-        return failed == 0
+        return result
     
     def _handle_partitioned_upload(self, db_name: str, base_table_name: str,
-                                  data: List[Dict], set_typ: Dict[str, str],
+                                  data: Iterator[List[Dict]], 
+                                  set_typ: Dict[str, str],
                                   partition_by: str, partition_date_column: str,
                                   allow_null: bool, update_on_duplicate: bool,
-                                  unique_keys: Optional[List[List[str]]]) -> bool:
+                                  unique_keys: Optional[List[List[str]]]) -> Dict[str, Any]:
         """处理分表上传"""
-        # 按日期分区数据
-        partitioned_data = DataProcessor.partition_data_by_date(
-            data, partition_date_column, partition_by
-        )
+        result = {
+            'inserted_rows': 0,
+            'skipped_rows': 0,
+            'failed_rows': 0,
+            'tables_created': []
+        }
         
-        total_success = True
+        # 使用更小的缓冲区，更频繁地刷新
+        partition_buffers = {}
+        buffer_limit = 1000  # 减小缓冲区大小
         
-        for partition_suffix, partition_data in partitioned_data.items():
-            partition_table_name = f"{base_table_name}_{partition_suffix}"
+        # 记录已创建的表，避免重复检查
+        created_tables = set()
+        
+        for chunk in data:
+            if not chunk:
+                continue
             
-            success = self._handle_single_table_upload(
-                db_name, partition_table_name, partition_data, set_typ,
-                allow_null, update_on_duplicate, unique_keys
+            # 按日期分区当前chunk
+            partitioned_chunk = DataProcessor.partition_data_by_date(
+                chunk, partition_date_column, partition_by
             )
             
-            if not success:
-                total_success = False
+            # 将数据添加到对应分区缓冲区
+            for partition_suffix, partition_data in partitioned_chunk.items():
+                if partition_suffix not in partition_buffers:
+                    partition_buffers[partition_suffix] = []
+                partition_buffers[partition_suffix].extend(partition_data)
+                
+                # 更频繁地刷新缓冲区
+                if len(partition_buffers[partition_suffix]) >= buffer_limit:
+                    partition_result = self._process_partition_buffer_optimized(
+                        db_name, base_table_name, partition_suffix,
+                        partition_buffers[partition_suffix], set_typ,
+                        allow_null, update_on_duplicate, unique_keys, created_tables
+                    )
+                    self._merge_partition_result(result, partition_result)
+                    partition_buffers[partition_suffix] = []  # 清空缓冲区
+            
+            # 定期检查所有缓冲区，防止某些分区数据积累过多
+            total_buffered = sum(len(buffer) for buffer in partition_buffers.values())
+            if total_buffered > 5000:  # 总缓冲超过5000条时强制刷新
+                for partition_suffix in list(partition_buffers.keys()):
+                    if partition_buffers[partition_suffix]:
+                        partition_result = self._process_partition_buffer_optimized(
+                            db_name, base_table_name, partition_suffix,
+                            partition_buffers[partition_suffix], set_typ,
+                            allow_null, update_on_duplicate, unique_keys, created_tables
+                        )
+                        self._merge_partition_result(result, partition_result)
+                        partition_buffers[partition_suffix] = []
+        
+        # 处理剩余的缓冲区数据
+        for partition_suffix, buffer_data in partition_buffers.items():
+            if buffer_data:
+                partition_result = self._process_partition_buffer_optimized(
+                    db_name, base_table_name, partition_suffix,
+                    buffer_data, set_typ, allow_null, update_on_duplicate, unique_keys, created_tables
+                )
+                self._merge_partition_result(result, partition_result)
         
         logger.info('分表上传完成', {
             '数据库': db_name,
             '基础表名': base_table_name,
-            '分区数': len(partitioned_data),
-            '总体成功': total_success
+            '分区数': len(created_tables),
+            '插入': result['inserted_rows'],
+            '跳过': result['skipped_rows'],
+            '失败': result['failed_rows']
         })
         
-        return total_success
+        return result
+    
+    def _process_partition_buffer_optimized(self, db_name: str, base_table_name: str,
+                                          partition_suffix: str, partition_data: List[Dict],
+                                          set_typ: Dict[str, str], allow_null: bool,
+                                          update_on_duplicate: bool, 
+                                          unique_keys: Optional[List[List[str]]],
+                                          created_tables: set) -> Dict[str, Any]:
+        """处理单个分区的缓冲数据"""
+        partition_table_name = f"{base_table_name}_{partition_suffix}"
+        
+        result = {
+            'inserted_rows': 0,
+            'skipped_rows': 0,
+            'failed_rows': 0,
+            'tables_created': []
+        }
+        
+        # 优化表存在性检查
+        table_key = f"{db_name}.{partition_table_name}"
+        if table_key not in created_tables:
+            if not self.table_mgr.table_exists(db_name, partition_table_name):
+                self.table_mgr.create_table(db_name, partition_table_name, set_typ, 
+                                           unique_keys=unique_keys)
+                result['tables_created'].append(table_key)
+            else:
+                # 表已存在，确保有时间戳列（但保持原有主键结构）
+                self.table_mgr.ensure_system_columns(db_name, partition_table_name)
+            created_tables.add(table_key)
+        
+        # 准备并插入数据
+        prepared_data = DataProcessor.prepare_data_for_insert(
+            partition_data, set_typ, allow_null
+        )
+        
+        inserted, skipped, failed = self.data_inserter.insert_data(
+            db_name, partition_table_name, prepared_data, set_typ, update_on_duplicate
+        )
+        
+        result['inserted_rows'] = inserted
+        result['skipped_rows'] = skipped
+        result['failed_rows'] = failed
+        
+        return result
+    
+    def _merge_partition_result(self, main_result: Dict[str, Any], 
+                               partition_result: Dict[str, Any]):
+        """合并分区处理结果"""
+        main_result['inserted_rows'] += partition_result['inserted_rows']
+        main_result['skipped_rows'] += partition_result['skipped_rows']
+        main_result['failed_rows'] += partition_result['failed_rows']
+        main_result['tables_created'].extend(partition_result['tables_created'])
     
     def close(self):
         """关闭连接"""
@@ -846,6 +1322,165 @@ class MySQLUploader:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+    def upload_data_concurrent(self, db_name: str, table_name: str, 
+                              data: Union[Dict, List[Dict], pd.DataFrame],
+                              set_typ: Optional[Dict[str, str]] = None,
+                              allow_null: bool = False,
+                              partition_by: Optional[str] = None,
+                              partition_date_column: str = '日期',
+                              update_on_duplicate: bool = False,
+                              unique_keys: Optional[List[List[str]]] = None,
+                              max_workers: int = 3) -> Dict[str, Any]:
+        """
+        并发上传数据到MySQL数据库
+        
+        :param max_workers: 最大并发工作线程数
+        :return: 上传结果详情
+        """
+        db_name = db_name.lower()
+        table_name = table_name.lower()
+        
+        result = {
+            'success': False,
+            'inserted_rows': 0,
+            'skipped_rows': 0,
+            'failed_rows': 0,
+            'tables_created': []
+        }
+        
+        try:
+            # 标准化数据为流式迭代器
+            normalized_data = DataProcessor.normalize_data(data, chunk_size=2000)  # 更小的chunk用于并发
+            
+            # 推断或验证列类型
+            if set_typ is None:
+                first_chunk = next(iter(normalized_data))
+                set_typ = DataTypeInferrer.infer_types_from_data(first_chunk)
+                normalized_data = DataProcessor.normalize_data(data, chunk_size=2000)
+                logger.debug('自动推断数据类型', {'类型映射': set_typ})
+            
+            # 确保数据库存在
+            self.table_mgr.ensure_database_exists(db_name)
+            
+            # 创建线程锁用于表创建的线程安全
+            table_creation_lock = threading.Lock()
+            created_tables_set = set()
+            
+            def process_chunk_worker(chunk_data):
+                """工作线程函数"""
+                try:
+                    if partition_by:
+                        # 分表处理
+                        partitioned_chunk = DataProcessor.partition_data_by_date(
+                            chunk_data, partition_date_column, partition_by
+                        )
+                        
+                        chunk_result = {
+                            'inserted_rows': 0,
+                            'skipped_rows': 0,
+                            'failed_rows': 0,
+                            'tables_created': []
+                        }
+                        
+                        for partition_suffix, partition_data in partitioned_chunk.items():
+                            partition_table_name = f"{table_name}_{partition_suffix}"
+                            table_key = f"{db_name}.{partition_table_name}"
+                            
+                            # 确保表存在（线程安全）
+                            with table_creation_lock:
+                                if table_key not in created_tables_set:
+                                    if not self.table_mgr.table_exists(db_name, partition_table_name):
+                                        self.table_mgr.create_table(db_name, partition_table_name, set_typ, 
+                                                                   unique_keys=unique_keys)
+                                        chunk_result['tables_created'].append(table_key)
+                                    else:
+                                        self.table_mgr.ensure_system_columns(db_name, partition_table_name)
+                                    created_tables_set.add(table_key)
+                            
+                            # 准备并插入数据
+                            prepared_data = DataProcessor.prepare_data_for_insert(
+                                partition_data, set_typ, allow_null
+                            )
+                            
+                            inserted, skipped, failed = self.data_inserter.insert_data(
+                                db_name, partition_table_name, prepared_data, set_typ, update_on_duplicate
+                            )
+                            
+                            chunk_result['inserted_rows'] += inserted
+                            chunk_result['skipped_rows'] += skipped
+                            chunk_result['failed_rows'] += failed
+                    else:
+                        # 单表处理
+                        table_key = f"{db_name}.{table_name}"
+                        with table_creation_lock:
+                            if table_key not in created_tables_set:
+                                if not self.table_mgr.table_exists(db_name, table_name):
+                                    self.table_mgr.create_table(db_name, table_name, set_typ, 
+                                                               unique_keys=unique_keys)
+                                    chunk_result = {'tables_created': [table_key]}
+                                else:
+                                    self.table_mgr.ensure_system_columns(db_name, table_name)
+                                    chunk_result = {'tables_created': []}
+                                created_tables_set.add(table_key)
+                            else:
+                                chunk_result = {'tables_created': []}
+                        
+                        prepared_chunk = DataProcessor.prepare_data_for_insert(
+                            chunk_data, set_typ, allow_null
+                        )
+                        
+                        inserted, skipped, failed = self.data_inserter.insert_data(
+                            db_name, table_name, prepared_chunk, set_typ, update_on_duplicate
+                        )
+                        
+                        chunk_result.update({
+                            'inserted_rows': inserted,
+                            'skipped_rows': skipped,
+                            'failed_rows': failed
+                        })
+                    
+                    return chunk_result
+                    
+                except Exception as e:
+                    logger.error('并发处理chunk失败', {'错误': str(e)})
+                    return {
+                        'inserted_rows': 0,
+                        'skipped_rows': 0,
+                        'failed_rows': len(chunk_data) if chunk_data else 0,
+                        'tables_created': []
+                    }
+            
+            # 使用线程池执行并发处理
+            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+                # 提交所有任务
+                future_to_chunk = {}
+                for chunk in normalized_data:
+                    if chunk:
+                        future = executor.submit(process_chunk_worker, chunk)
+                        future_to_chunk[future] = len(chunk)
+                
+                # 收集结果
+                for future in concurrent.futures.as_completed(future_to_chunk):
+                    chunk_result = future.result()
+                    result['inserted_rows'] += chunk_result['inserted_rows']
+                    result['skipped_rows'] += chunk_result['skipped_rows']
+                    result['failed_rows'] += chunk_result['failed_rows']
+                    result['tables_created'].extend(chunk_result['tables_created'])
+            
+            # 去重tables_created
+            result['tables_created'] = list(set(result['tables_created']))
+            result['success'] = result['failed_rows'] == 0
+            
+        except Exception as e:
+            logger.error('并发数据上传失败', {
+                '数据库': db_name,
+                '表名': table_name,
+                '错误': str(e)
+            })
+            result['success'] = False
+        
+        return result
+
 
 # 使用示例
 if __name__ == '__main__':
@@ -863,24 +1498,14 @@ if __name__ == '__main__':
         {'name': 'Bob', 'age': 30, 'salary': 60000.0, '日期': '2023-01-02'},
     ]
     
-    # 定义列类型（系统会自动添加id、create_at、update_at列）
-    column_types = {
-        'name': 'VARCHAR(255)',
-        'age': 'INT',
-        'salary': 'DECIMAL(10,2)',
-        '日期': 'DATE'
-    }
-    
-    # 上传数据
-    success = uploader.upload_data(
+    # 上传数据（自动推断类型，流式处理）
+    result = uploader.upload_data(
         db_name='test_db',
         table_name='test_table',
         data=sample_data,
-        set_typ=column_types,
-        allow_null=False,
         update_on_duplicate=True,
         unique_keys=[['name', '日期']]
     )
     
     uploader.close()
-    print(f"上传结果: {success}") 
+    print(f"上传结果: {result}") 

mdbq/other/download_sku_picture.py

@@ -820,18 +820,11 @@ def main(service_name, database):
                 db_name='属性设置2',
                 table_name='天猫商品sku信息',
                 data=s.df,
-                set_typ={},  # 定义列和数据类型
-                primary_keys=[],  # 创建唯一主键
-                check_duplicate=False,  # 检查重复数据
-                update_on_duplicate=False,  # 遇到重复时更新数据，默认 False 跳过
-                duplicate_columns=[],  # 指定排重的组合键
-                allow_null=False,  # 允许插入空值
-                partition_by=None,  # 按年/月分表
-                partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
-                auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
-                indexes=[],  # 指定索引列
-                transaction_mode='row',  # 事务模式
-                unique_keys=[[]],  # 唯一约束列表
+                set_typ=None,
+                allow_null=False,
+                partition_by=None,
+                update_on_duplicate=True,
+                unique_keys=None,
             )
 
 
@@ -907,18 +900,11 @@ def download_sku(service_name='company', database='mysql', db_name='属性设置
                 db_name=table_name,
                 table_name=table_name,
                 data=s.df,
-                set_typ={},  # 定义列和数据类型
-                primary_keys=[],  # 创建唯一主键
-                check_duplicate=False,  # 检查重复数据
-                update_on_duplicate=False,  # 遇到重复时更新数据，默认 False 跳过
-                duplicate_columns=[],  # 指定排重的组合键
-                allow_null=False,  # 允许插入空值
-                partition_by=None,  # 按年/月分表
-                partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
-                auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
-                indexes=[],  # 指定索引列
-                transaction_mode='row',  # 事务模式
-                unique_keys=[[]],  # 唯一约束列表
+                set_typ=None,
+                allow_null=False,
+                partition_by=None,
+                update_on_duplicate=True,
+                unique_keys=None,
             )
 
     # 从数据库中读取数据，并下载素材到本地
@@ -954,18 +940,11 @@ def download_sku(service_name='company', database='mysql', db_name='属性设置
             db_name=db_name,
             table_name=table_name,
             data=df,
-            set_typ={},  # 定义列和数据类型
-            primary_keys=[],  # 创建唯一主键
-            check_duplicate=False,  # 检查重复数据
-            update_on_duplicate=False,  # 遇到重复时更新数据，默认 False 跳过
-            duplicate_columns=[],  # 指定排重的组合键
-            allow_null=False,  # 允许插入空值
-            partition_by=None,  # 按年/月分表
-            partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
-            auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
-            indexes=[],  # 指定索引列
-            transaction_mode='row',  # 事务模式
-            unique_keys=[[]],  # 唯一约束列表
+            set_typ=None,
+            allow_null=False,
+            partition_by=None,
+            update_on_duplicate=True,
+            unique_keys=None,
         )
 
 

{mdbq-4.2.0.dist-info → mdbq-4.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.2.0
+Version: 4.2.2
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.2.0.dist-info → mdbq-4.2.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=ZxnKNunFTlHOzO3MsfFUVCuATEizUGyGn-xxae0gXSI,17
+mdbq/__version__.py,sha256=gNDA6f7PmXcbqB0lTY4HIgD6dEB6SGywjhHa3HAyczA,17
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
 mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -15,9 +15,9 @@ mdbq/mysql/deduplicator.py,sha256=tzLIm9K9S0lGLlVTI0dDQVYpWX796XCuyufmw1lU26Y,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=N2xHJf2CiUXjXIVBemdst-wamIP3908EGAJOFG13fCU,50475
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=BFJnrXvQYU7soZMr-vm3zChhqmw960eePOy8QqebRDo,32678
+mdbq/mysql/uploader.py,sha256=2inrXu3PIlvowfm5_0U4Trx_mraApjII8g_5ycFbNJ0,60059
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
+mdbq/other/download_sku_picture.py,sha256=MJX47I9jTUMFzO1kyEH-onIzAGa6QpgfmghrmyYnEsc,45111
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.2.0.dist-info/METADATA,sha256=vyR4bK1uDkWIH2Y1t-0cmuRB8wWcw59xiAvv_EZw5_I,363
-mdbq-4.2.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.2.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.2.0.dist-info/RECORD,,
+mdbq-4.2.2.dist-info/METADATA,sha256=vfhvk7DXQ267-NOPdqKJ_AWCWSEbWKdDjIf7bilbCXo,363
+mdbq-4.2.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.2.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.2.2.dist-info/RECORD,,