mdbq 4.0.8__py3-none-any.whl → 4.0.10__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.8'
+VERSION = '4.0.10'

mdbq/aggregation/query_data.py

@@ -119,9 +119,16 @@ def upload_data_decorator(**upload_kwargs):
                     return None
 
                 # 处理 DataFrame 结果
-                if isinstance(result, pd.DataFrame):
+                if isinstance(result, (pd.DataFrame, list, dict)):
                     if set_type is not None:
-                        result = reorder_columns(result, set_type)
+                        if isinstance(result, pd.DataFrame):
+                                result = reorder_columns(result, set_type)
+                        elif isinstance(result, list):
+                            # 如果是list，转换为DataFrame以调整列顺序
+                            result = reorder_columns(pd.DataFrame(result), set_type)
+                        elif isinstance(result, dict):
+                            # 如果是dict，转换为DataFrame以调整列顺序
+                            result = reorder_columns(pd.DataFrame([result]), set_type)
                     
                     # 合并参数
                     merged_kwargs = {
@@ -143,12 +150,19 @@ def upload_data_decorator(**upload_kwargs):
 
                     df, extra_kwargs = result[0], result[1]
                     
-                    if not isinstance(df, pd.DataFrame):
-                        logger.warning('函数返回的元组第一个元素不是DataFrame，直接返回原结果，不执行上传', {'函数': func.__name__, '库': db_name, '表': table_name})
+                    if not isinstance(df, (pd.DataFrame, list, dict)):
+                        logger.warning('函数返回的元组第一个元素不是DataFrame/list/dict，直接返回原结果，不执行上传', {'函数': func.__name__, '库': db_name, '表': table_name})
                         return result
 
                     if set_type is not None:
-                        df = reorder_columns(df, set_type)
+                        if isinstance(df, pd.DataFrame):
+                                df = reorder_columns(df, set_type)
+                        elif isinstance(df, list):
+                            # 如果是list，转换为DataFrame以调整列顺序
+                            df = reorder_columns(pd.DataFrame(df), set_type)
+                        elif isinstance(df, dict):
+                            # 如果是dict，转换为DataFrame以调整列顺序
+                            df = reorder_columns(pd.DataFrame([df]), set_type)
                         result = (df, extra_kwargs) + result[2:]
 
                     # 合并参数
@@ -1543,7 +1557,6 @@ class MysqlDatasQuery:
             'unique_keys': [['日期', '店铺id', '商品id']],  # 唯一约束列表
         }
 
-
     @upload_data_decorator()
     def spph(self, db_name='聚合数据', table_name='天猫_商品排行'):
         """  """
@@ -2370,61 +2383,57 @@ class MysqlDatasQuery:
             '更新时间': 'timestamp',
         }
         logger.info('正在更新数据库', {'主机': f'{host}:{port}', '库': db_name, '表': table_name})
-        new_dict = {
-            '日期': '',
-            '店铺名称': '',
-            '场次信息': '',
-            '场次id': '',
-            '直播开播时间': '',
-            '开播时长': '',
-            '封面图点击率': '',
-            '观看人数': '',
-            '观看次数': '',
-            '新增粉丝数': '',
-            '流量券消耗': '',
-            '观看总时长': '',
-            '人均观看时长': '',
-            '次均观看时长': '',
-            '商品点击人数': '',
-            '商品点击次数': '',
-            '商品点击率': '',
-            '加购人数': '',
-            '加购件数': '',
-            '加购次数': '',
-            '成交金额': '',
-            '成交人数': '',
-            '成交件数': '',
-            '成交笔数': '',
-            '成交转化率': '',
-            '退款人数': '',
-            '退款笔数': '',
-            '退款件数': '',
-            '退款金额': '',
-            '预售定金支付金额': '',
-            '预售预估总金额': '',
-        }
-        _results = []
-        for dict_data in df.to_dict(orient='records'):
-            new_dict.update(dict_data)
-            _results.append(new_dict)
-        if _results:
-            return _results, {
-                'db_name': db_name,
-                'table_name': table_name,
-                'set_typ': set_typ,
-                'primary_keys': ['场次id'],  # 创建唯一主键
-                'check_duplicate': False,  # 检查重复数据
-                'duplicate_columns': [],  # 指定排重的组合键
-                'update_on_duplicate': True,  # 更新旧数据
-                'allow_null': False,  # 允许插入空值
-                'partition_by': None,  # 分表方式
-                'partition_date_column': '日期',  # 用于分表的日期列名，默认为'日期'
-                'indexes': [],  # 普通索引列
-                'transaction_mode': 'batch',  # 事务模式
-                'unique_keys': [['场次id']],  # 唯一约束列表
-            }
-        else:
-            return None, None
+        ordered_columns = [
+            '日期',
+            '店铺名称',
+            '场次信息',
+            '场次id',
+            '直播开播时间',
+            '开播时长',
+            '封面图点击率',
+            '观看人数',
+            '观看次数',
+            '新增粉丝数',
+            '流量券消耗',
+            '观看总时长',
+            '人均观看时长',
+            '次均观看时长',
+            '商品点击人数',
+            '商品点击次数',
+            '商品点击率',
+            '加购人数',
+            '加购件数',
+            '加购次数',
+            '成交金额',
+            '成交人数',
+            '成交件数',
+            '成交笔数',
+            '成交转化率',
+            '退款人数',
+            '退款笔数',
+            '退款件数',
+            '退款金额',
+            '预售定金支付金额',
+            '预售预估总金额',
+        ]
+        # 使用reindex重排列顺序，未定义的列会自动放在最后
+        df = df.reindex(columns=[col for col in ordered_columns if col in df.columns] + 
+                              [col for col in df.columns if col not in ordered_columns])
+        return df, {
+            'db_name': db_name,
+            'table_name': table_name,
+            'set_typ': set_typ,
+            'primary_keys': [],  # 创建唯一主键
+            'check_duplicate': False,  # 检查重复数据
+            'duplicate_columns': [],  # 指定排重的组合键
+            'update_on_duplicate': True,  # 更新旧数据
+            'allow_null': False,  # 允许插入空值
+            'partition_by': None,  # 分表方式
+            'partition_date_column': '日期',  # 用于分表的日期列名，默认为'日期'
+            'indexes': [],  # 普通索引列
+            'transaction_mode': 'batch',  # 事务模式
+            'unique_keys': [['场次id']],  # 唯一约束列表
+        }
 
     # @try_except
     @upload_data_decorator()
@@ -3675,13 +3684,25 @@ def main(months=3):
         password=password, 
         host=host, 
         port=port,
-        maxconnections=30,
+        maxconnections=20,
         )
     query1(download_manager=download_manager, months=months)
     query2(download_manager=download_manager, months=months)
     query3(download_manager=download_manager, months=months)
+    logger.info('数据聚合完成')
 
 
 if __name__ == '__main__':
     main(months=3)
     pass
+
+    # download_manager = s_query.QueryDatas(
+    #     username=username, 
+    #     password=password, 
+    #     host=host, 
+    #     port=port,
+    #     maxconnections=10,
+    #     )
+    # sdq = MysqlDatasQuery(download_manager=download_manager) 
+    # sdq.months = 3
+    # sdq.zb_ccfx(db_name='聚合数据', table_name='生意参谋_直播场次分析')

mdbq/mysql/s_query.py

@@ -47,9 +47,9 @@ class QueryDatas:
             host: 数据库主机
             port: 数据库端口
             charset: 字符集，默认utf8mb4
-            maxconnections: 最大连接数，默认20
-            mincached: 最小缓存连接数，默认2
-            maxcached: 最大缓存连接数，默认5
+            maxconnections: 最大活动连接数，默认20
+            mincached: 最小缓存连接数，空闲连接数量，默认2
+            maxcached: 最大缓存连接数，最大空闲连接数，默认5
             connect_timeout: 连接超时时间，默认10秒
             read_timeout: 读取超时时间，默认30秒
             write_timeout: 写入超时时间，默认30秒
@@ -253,20 +253,8 @@ class QueryDatas:
 
     # @_execute_with_retry
     def _get_connection(self, db_name: Optional[str] = None) -> pymysql.connections.Connection:
-        """
-        从连接池获取数据库连接
-        
-        Args:
-            db_name: 可选的数据库名，如果提供则会在连接后选择该数据库
-            
-        Returns:
-            数据库连接对象
-            
-        Raises:
-            ConnectionError: 当获取连接失败时抛出
-        """
+        """从连接池获取数据库连接"""
         try:
-            # 只在连续失败次数达到阈值时检查健康状态
             if self._pool_stats['consecutive_failures'] >= self._pool_stats['max_consecutive_failures']:
                 if not self._check_pool_health():
                     logger.warning('连接池不健康，尝试重新创建')
@@ -282,66 +270,184 @@ class QueryDatas:
             error_code = e.args[0] if e.args else None
             if error_code in (2003, 2006, 2013):
                 logger.error('数据库连接错误', {
+                    '库': db_name,
                     '错误代码': error_code,
                     '错误信息': str(e),
-                    '数据库': db_name
                 })
                 self.pool = self._create_connection_pool(10, 2, 5)
                 self._pool_stats['consecutive_failures'] = 0
                 raise ConnectionError(f'数据库连接错误: {str(e)}')
-            else:
-                raise
+            raise
         except Exception as e:
             logger.error('从连接池获取数据库连接失败', {
+                '库': db_name,
                 '错误': str(e),
-                '数据库': db_name
             })
             raise ConnectionError(f'连接数据库失败: {str(e)}')
 
     # @_execute_with_retry
-    def _execute_query(self, sql: str, params: tuple = None, db_name: str = None) -> Optional[List[Dict[str, Any]]]:
+    def _execute_query(self, sql: str, params: tuple = None, db_name: str = None, 
+                      fetch_all: bool = True, error_handling: bool = True) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
+        """执行SQL查询的通用方法"""
+        try:
+            if sql.upper().startswith('SHOW DATABASES'):
+                with closing(self._get_connection()) as connection:
+                    with closing(connection.cursor()) as cursor:
+                        cursor.execute(sql, params)
+                        return cursor.fetchall() if fetch_all else cursor.fetchone()
+            else:
+                with closing(self._get_connection(db_name)) as connection:
+                    with closing(connection.cursor()) as cursor:
+                        cursor.execute(sql, params)
+                        return cursor.fetchall() if fetch_all else cursor.fetchone()
+        except pymysql.OperationalError as e:
+            error_code = e.args[0] if e.args else None
+            if error_handling:
+                if error_code in (1045, 1049):  # 访问被拒绝或数据库不存在
+                    logger.error('数据库访问错误', {
+                        'SQL': sql,
+                        '参数': params,
+                        '库': db_name,
+                        '错误代码': error_code,
+                        '错误信息': str(e)
+                    })
+                else:
+                    logger.error('数据库操作错误', {
+                        '库': db_name,
+                        'SQL': sql,
+                        '参数': params,
+                        '错误代码': error_code,
+                        '错误信息': str(e)
+                    })
+                return None
+            raise
+        except Exception as e:
+            if error_handling:
+                logger.error('执行SQL查询失败', {
+                    '库': db_name,
+                    'SQL': sql,
+                    '参数': params,
+                    '错误类型': type(e).__name__,
+                    '错误信息': str(e)
+                })
+                return None
+            raise
+
+    def _get_table_info(self, db_name: str, table_name: str, info_type: Literal['columns', 'dtypes', 'exists'] = 'exists') -> Union[bool, List[Dict[str, Any]], List[str]]:
         """
-        执行SQL查询的通用方法。
+        获取表信息的通用方法。
         
         Args:
-            sql: SQL查询语句
-            params: 查询参数
             db_name: 数据库名
-            
+            table_name: 表名
+            info_type: 信息类型
+                - 'exists': 检查表是否存在（默认）
+                - 'columns': 获取列名列表
+                - 'dtypes': 获取列名和类型
+                
         Returns:
-            查询结果列表，如果查询失败返回None
+            根据info_type返回不同类型的信息：
+            - 'exists': 返回bool，表示表是否存在
+            - 'columns': 返回列名列表
+            - 'dtypes': 返回列名和类型的列表
         """
         try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    cursor.execute(sql, params)
-                    return cursor.fetchall()
+            if info_type == 'exists':
+                result = self._execute_query("SHOW DATABASES LIKE %s", (db_name,))
+                if not result:
+                    all_dbs = self._execute_query("SHOW DATABASES")
+                    available_dbs = [db['Database'] for db in all_dbs] if all_dbs else []
+                    logger.info('数据库不存在', {
+                        '库': db_name,
+                        '可用的数据库': available_dbs,
+                        '可能的原因': '数据库名称错误或没有访问权限'
+                    })
+                    return False
+                    
+                result = self._execute_query("SHOW TABLES LIKE %s", (table_name,), db_name=db_name)
+                if not result:
+                    all_tables = self._execute_query("SHOW TABLES", db_name=db_name)
+                    available_tables = [table[f'Tables_in_{db_name}'] for table in all_tables] if all_tables else []
+                    logger.info('表不存在', {
+                        '库': db_name,
+                        '表': table_name,
+                        '可用的表': available_tables,
+                        '可能的原因': '表名称错误或没有访问权限'
+                    })
+                    return False
+                return True
+                
+            elif info_type == 'columns':
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                result = self._execute_query(sql, (db_name, table_name))
+                return [col['COLUMN_NAME'] for col in result] if result else []
+                
+            elif info_type == 'dtypes':
+                sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                return self._execute_query(sql, (db_name, table_name)) or []
+                
         except Exception as e:
-            logger.error('执行SQL查询失败', {
-                'SQL': sql,
-                '参数': params,
-                '数据库': db_name,
+            logger.error('获取表信息失败', {
+                '库': db_name,
+                '表': table_name,
+                '信息类型': info_type,
                 '错误类型': type(e).__name__,
                 '错误信息': str(e)
             })
-            return None
+            return [] if info_type != 'exists' else False
 
-    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
-        """
-        按指定条件查询数据库表，返回满足条件的指定字段数据。
+    def check_infos(self, db_name: str, table_name: str) -> bool:
+        """检查数据库和数据表是否存在"""
+        return self._get_table_info(db_name, table_name, 'exists')
+
+    def _format_columns(self, columns: List[str]) -> str:
+        """格式化列名列表为SQL语句"""
+        return ', '.join([f'`{col}`' for col in columns])
+
+    def columns_to_list(self, db_name: str, table_name: str, columns_name: List[str], where: str = None) -> List[Dict[str, Any]]:
+        """获取数据表的指定列数据"""
+        if not self._get_table_info(db_name, table_name):
+            return []
         
-        Args:
-            db_name: 数据库名
-            table_name: 表名
-            condition: SQL条件字符串（不含WHERE）
-            columns: 查询字段字符串或以逗号分隔的字段名，默认'更新时间'
+        try:
+            existing_columns = self._get_table_info(db_name, table_name, 'columns')
+            columns_name = [col for col in columns_name if col in existing_columns]
             
-        Returns:
-            查询结果列表，如果查询失败返回None
-        """
-        if not self.check_infos(db_name, table_name):
-            return None
+            if not columns_name:
+                logger.info('未找到匹配的列名', {'库': db_name, '表': table_name, '请求列': columns_name})
+                return []
+            
+            sql = f"SELECT {self._format_columns(columns_name)} FROM `{db_name}`.`{table_name}`"
+            if where:
+                sql += f" WHERE {where}"
             
+            logger.debug('执行列查询', {'库': db_name, '表': table_name, 'SQL': sql})
+            return self._execute_query(sql, db_name=db_name) or []
+            
+        except Exception as e:
+            logger.error('列查询失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
+            return []
+
+    def dtypes_to_list(self, db_name: str, table_name: str, columns_name: List[str] = None) -> List[Dict[str, Any]]:
+        """获取数据表的列名和类型"""
+        if not self._get_table_info(db_name, table_name):
+            return []
+        
+        try:
+            result = self._get_table_info(db_name, table_name, 'dtypes')
+            if columns_name:
+                columns_name = set(columns_name)
+                result = [row for row in result if row['COLUMN_NAME'] in columns_name]
+            return result
+        except Exception as e:
+            logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
+            return []
+
+    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
+        """按指定条件查询数据库表"""
+        if not self._get_table_info(db_name, table_name):
+            return None
+        
         sql = f"SELECT {columns} FROM `{table_name}` WHERE {condition}"
         logger.debug('执行SQL查询', {'库': db_name, '表': table_name, 'SQL': sql})
         return self._execute_query(sql, db_name=db_name)
@@ -598,98 +704,6 @@ class QueryDatas:
                 df[col] = df[col].astype(float)
         return df
 
-    # @_execute_with_retry
-    def columns_to_list(self, db_name, table_name, columns_name, where: str = None) -> list:
-        """
-        获取数据表的指定列, 支持where条件筛选, 返回列表字典。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns_name: 需要获取的列名列表
-        :param where: 可选，SQL条件字符串（不含WHERE）
-        :return: [{列1:值, 列2:值, ...}, ...]
-        """
-        if not self.check_infos(db_name, table_name):
-            return []
-            
-        try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, table_name))
-                    cols_exist = [col['COLUMN_NAME'] for col in cursor.fetchall()]
-                    columns_name = [item for item in columns_name if item in cols_exist]
-                    if not columns_name:
-                        logger.info('未找到匹配的列名', {'库': db_name, '表': table_name, '请求列': columns_name})
-                        return []
-                    columns_in = ', '.join([f'`{col}`' for col in columns_name])
-                    sql = f"SELECT {columns_in} FROM `{db_name}`.`{table_name}`"
-                    if where:
-                        sql += f" WHERE {where}"
-                    logger.debug('执行列查询', {'库': db_name, '表': table_name, 'SQL': sql})
-                    cursor.execute(sql)
-                    column_values = cursor.fetchall()
-            return column_values
-        except Exception as e:
-            logger.error('列查询失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
-            return []
-
-    # @_execute_with_retry
-    def dtypes_to_list(self, db_name, table_name, columns_name=None) -> list:
-        """
-        获取数据表的列名和类型, 支持只返回部分字段类型。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param columns_name: 可选，字段名列表，仅返回这些字段的类型
-        :return: [{'COLUMN_NAME': ..., 'COLUMN_TYPE': ...}, ...]
-        """
-        if not self.check_infos(db_name, table_name):
-            return []
-            
-        try:
-            with closing(self._get_connection(db_name)) as connection:
-                with closing(connection.cursor()) as cursor:
-                    sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, table_name))
-                    column_name_and_type = cursor.fetchall()
-                    if columns_name:
-                        columns_name = set(columns_name)
-                        column_name_and_type = [row for row in column_name_and_type if row['COLUMN_NAME'] in columns_name]
-            return column_name_and_type
-        except Exception as e:
-            logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
-            return []
-
-    # @_execute_with_retry
-    def check_infos(self, db_name, table_name) -> bool:
-        """
-        检查数据库和数据表是否存在。
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :return: 存在返回True，否则False
-        """
-        try:
-            # 检查数据库是否存在
-            result = self._execute_query("SHOW DATABASES LIKE %s", (db_name,))
-            if not result:
-                logger.info('数据库不存在', {'库': db_name})
-                return False
-                
-            # 检查表是否存在
-            result = self._execute_query("SHOW TABLES LIKE %s", (table_name,), db_name=db_name)
-            if not result:
-                logger.info('表不存在', {'库': db_name, '表': table_name})
-                return False
-            return True
-            
-        except Exception as e:
-            logger.error('检查数据库或表失败', {
-                '库': db_name,
-                '表': table_name,
-                '错误类型': type(e).__name__,
-                '错误信息': str(e)
-            })
-            return False
-
     def __enter__(self):
         """上下文管理器入口"""
         return self
@@ -772,6 +786,8 @@ class QueryDatas:
             - 当return_format='list_dict'时，返回列表字典
             - 如果查询失败，返回空的DataFrame或空列表
         """
+        start_time = time.time()
+        
         if not db_name or not table_name:
             logger.error('数据库名和表名不能为空', {'库': db_name, '表': table_name})
             return [] if return_format == 'list_dict' else pd.DataFrame()
@@ -786,7 +802,7 @@ class QueryDatas:
         start_date, end_date = self._validate_date_range(start_date, end_date, db_name, table_name)
         
         # 检查数据库和表是否存在
-        if not self.check_infos(db_name, table_name):
+        if not self._get_table_info(db_name, table_name):
             return [] if return_format == 'list_dict' else pd.DataFrame()
         try:
             with closing(self._get_connection(db_name)) as connection:
@@ -863,7 +879,7 @@ class QueryDatas:
                     target_time = 1.0  # 期望每批1秒
 
                     while offset < total_count:
-                        start_time = time.time()
+                        _p_time = time.time()
                         # 添加分页参数
                         page_sql = f"{base_sql} LIMIT %s OFFSET %s"
                         page_params = list(params) + [page_size, offset]
@@ -881,7 +897,7 @@ class QueryDatas:
                             else:
                                 all_results = pd.concat([all_results, pd.DataFrame(page_results)], ignore_index=True)
 
-                        duration = time.time() - start_time
+                        duration = time.time() - _p_time
                         page_size = self._adjust_page_size(duration, page_size, min_size, max_size, target_time)
                         offset += len(page_results)
                         logger.debug('分页查询进度', {
@@ -896,6 +912,21 @@ class QueryDatas:
 
                     if return_format == 'df' and isinstance(all_results, pd.DataFrame) and not all_results.empty:
                         all_results = self._convert_decimal_columns(all_results)
+                    logger.info('查询完成', {
+                        '库': db_name,
+                        '表': table_name,
+                        '总记录数': total_count,
+                        '已获取记录数': len(all_results) if return_format == 'list_dict' else len(all_results.index),
+                        '查询耗时': f'{time.time() - start_time:.2f}s',
+                        '查询参数': {
+                            '开始日期': start_date,
+                            '结束日期': end_date,
+                            '日期字段': date_field,
+                            '限制行数': limit,
+                            '分页大小': page_size,
+                            '返回数据格式': return_format,
+                        }
+                    })
                     return all_results
 
         except Exception as e:
@@ -903,7 +934,15 @@ class QueryDatas:
                 '库': db_name,
                 '表': table_name,
                 '错误类型': type(e).__name__,
-                '错误信息': str(e)
+                '错误信息': str(e),
+                '查询参数': {
+                    '开始日期': start_date,
+                    '结束日期': end_date,
+                    '日期字段': date_field,
+                    '限制行数': limit,
+                    '分页大小': page_size,
+                    '返回数据格式': return_format,
+                }
             })
             return [] if return_format == 'list_dict' else pd.DataFrame()
 

mdbq/mysql/uploader.py

@@ -404,7 +404,15 @@ class MySQLUploader:
             raise ValueError('set_typ 未指定')
         # set_typ的键清洗
         set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
-        column_defs = ["`id` INT NOT NULL AUTO_INCREMENT"]
+        
+        # 处理id列和主键
+        column_defs = []
+        
+        # 添加id列（仅在没有指定主键时）
+        if not primary_keys:
+            column_defs.append("`id` INT NOT NULL AUTO_INCREMENT")
+        
+        # 添加其他列
         for col_name, col_type in set_typ.items():
             if col_name == 'id':
                 continue
@@ -413,18 +421,23 @@ class MySQLUploader:
             if not allow_null and not col_type.lower().startswith('json'):
                 col_def += " NOT NULL"
             column_defs.append(col_def)
+            
         # 主键处理逻辑调整
         def _index_col_sql(col):
             col_type = set_typ.get(col, '').lower()
             if 'varchar' in col_type or 'text' in col_type:
                 return f"`{self._normalize_col(col)}`(100)"
             return f"`{self._normalize_col(col)}`"
+            
+        # 处理主键
         if primary_keys and len(primary_keys) > 0:
+            # 如果指定了主键，直接使用指定的主键
             safe_primary_keys = [_index_col_sql(pk) for pk in primary_keys]
             primary_key_sql = f"PRIMARY KEY ({','.join(safe_primary_keys)})"
         else:
-            safe_primary_keys = [_index_col_sql('id')]
+            # 如果没有指定主键，使用id作为主键
             primary_key_sql = f"PRIMARY KEY (`id`)"
+            
         # 索引统一在CREATE TABLE中定义
         index_defs = []
         if date_column and date_column in set_typ:
@@ -435,15 +448,28 @@ class MySQLUploader:
                 if idx_col in set_typ:
                     safe_idx_col = _index_col_sql(idx_col)
                     index_defs.append(f"INDEX `idx_{self._normalize_col(idx_col)}` ({safe_idx_col})")
+                    
         # UNIQUE KEY定义
         unique_defs = []
         if unique_keys:
             for unique_cols in unique_keys:
                 if not unique_cols:
                     continue
+                # 检查唯一约束是否与主键冲突
+                if primary_keys:
+                    # 如果唯一约束的列是主键的一部分，则跳过
+                    if set(unique_cols).issubset(set(primary_keys)):
+                        logger.warning('跳过与主键冲突的唯一约束', {
+                            '库': db_name, 
+                            '表': table_name, 
+                            '唯一约束': unique_cols, 
+                            '主键': primary_keys
+                        })
+                        continue
                 safe_unique_cols = [_index_col_sql(col) for col in unique_cols]
                 unique_name = f"uniq_{'_'.join([self._normalize_col(c) for c in unique_cols])}"
                 unique_defs.append(f"UNIQUE KEY `{unique_name}` ({','.join(safe_unique_cols)})")
+                
         index_defs = list(set(index_defs))
         all_defs = column_defs + [primary_key_sql] + index_defs + unique_defs
         sql = f"""
@@ -1437,11 +1463,22 @@ class MySQLUploader:
                         try:
                             cursor.executemany(sql, values_list)
                             conn.commit()
-                            inserted = cursor.rowcount if cursor.rowcount is not None else 0
-                            total_inserted += inserted
-                            total_skipped += len(batch) - inserted
+                            # 在batch模式下，affected_rows表示实际影响的行数
+                            # 如果update_on_duplicate为True，则affected_rows包含更新的行数
+                            # 如果update_on_duplicate为False，则affected_rows只包含插入的行数
+                            affected = cursor.rowcount if cursor.rowcount is not None else 0
+                            if update_on_duplicate:
+                                # 当启用更新时，affected_rows包含插入和更新的行数
+                                # 我们需要区分插入和更新的行数
+                                # 由于无法准确区分，我们假设所有行都是插入的
+                                total_inserted += len(batch)
+                            else:
+                                # 当不启用更新时，affected_rows只包含插入的行数
+                                total_inserted += affected
+                                total_skipped += len(batch) - affected
                         except pymysql.err.IntegrityError as e:
                             conn.rollback()
+                            # 在唯一约束冲突时，所有行都被跳过
                             total_skipped += len(batch)
                             logger.debug('批量插入唯一约束冲突，全部跳过', {'库': db_name, '表': table_name, '错误': str(e)})
                         except Exception as e:
@@ -1460,10 +1497,16 @@ class MySQLUploader:
                                     values += [row.get(col) for col in dup_cols]
                                 cursor.execute(sql, values)
                                 affected = cursor.rowcount if cursor.rowcount is not None else 0
-                                if affected > 0:
+                                if update_on_duplicate:
+                                    # 当启用更新时，affected_rows包含插入和更新的行数
+                                    # 假设所有行都是插入的，因为无法区分插入和更新
                                     total_inserted += 1
                                 else:
-                                    total_skipped += 1
+                                    # 当不启用更新时，affected_rows只包含插入的行数
+                                    if affected > 0:
+                                        total_inserted += 1
+                                    else:
+                                        total_skipped += 1
                             except pymysql.err.IntegrityError as e:
                                 conn.rollback()
                                 total_skipped += 1
@@ -1482,10 +1525,16 @@ class MySQLUploader:
                                 values += [row.get(col) for col in dup_cols]
                             cursor.execute(sql, values)
                             affected = cursor.rowcount if cursor.rowcount is not None else 0
-                            if affected > 0:
+                            if update_on_duplicate:
+                                # 当启用更新时，affected_rows包含插入和更新的行数
+                                # 假设所有行都是插入的，因为无法区分插入和更新
                                 total_inserted += 1
                             else:
-                                total_skipped += 1
+                                # 当不启用更新时，affected_rows只包含插入的行数
+                                if affected > 0:
+                                    total_inserted += 1
+                                else:
+                                    total_skipped += 1
                             conn.commit()
                         except pymysql.err.IntegrityError as e:
                             conn.rollback()

{mdbq-4.0.8.dist-info → mdbq-4.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.8
+Version: 4.0.10
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.8.dist-info → mdbq-4.0.10.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=ctSY0gLLZnr9k_avSjRnC3qI18wZwhq3iBz504pEU14,17
+mdbq/__version__.py,sha256=DkNrzYKkaEbmtMXcPR8H0qa5y2WJne8kPSNBfTO6mFo,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=86S_Iy3NNTkn-tNNLxlnwqDnXv3x3aHD1cCe1mL7jdg,165423
+mdbq/aggregation/query_data.py,sha256=NdhsLJvt6NgZSMHpkMxnmEEXzDhUiR5tRwYwI-PfwIw,166732
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -10,9 +10,9 @@ mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,16
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=FSFrFZE5yzEbnpLrN2AmlRZ_VvTvfpIWaQUjZfLIi9g,40342
+mdbq/mysql/s_query.py,sha256=jGBdGPE4mtB06vccfaWIEWpSAqdY-nWc1s9bzCUh8Gg,42916
 mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=OtLWoGflmndc7z0A1aK1Qj-XVkWtGyN4Y5YnLsRovJc,78215
+mdbq/mysql/uploader.py,sha256=wNQE7UjCEyAKri9CnQXO7d6EVXCaYqFze2i2tcGAVpw,81001
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.8.dist-info/METADATA,sha256=zmc_fzk3uaGju9Nby1Xvta_2po8NNOG7CM0TvvDIiuU,363
-mdbq-4.0.8.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.8.dist-info/RECORD,,
+mdbq-4.0.10.dist-info/METADATA,sha256=AJXVA5kCyLJQiObIW13jNvJAWsXdXJRFN2xCCSdDO78,364
+mdbq-4.0.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.10.dist-info/RECORD,,