mdbq 4.0.3__py3-none-any.whl → 4.0.4__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.3'
+VERSION = '4.0.4'

mdbq/aggregation/query_data.py

@@ -14,7 +14,8 @@ import platform
 import os
 import time
 import calendar
-import concurrent.futures
+from collections.abc import Mapping, Sequence
+import inspect
 
 dir_path = os.path.expanduser("~")
 config_file = os.path.join(dir_path, 'spd.txt')
@@ -36,6 +37,47 @@ logger = mylogger.MyLogger(
 )
 
 
+def reorder_columns(df: pd.DataFrame, set_type) -> pd.DataFrame:
+    """
+    调整DataFrame的列顺序，按照set_type中的顺序排列，忽略大小写，set_type中不存在的列自动跳过。
+    set_type可以是列表或字典（此时用字典的键名作为顺序）。
+    不改变数据和数据类型。
+    如果 set_type 为 None、空列表或空字典，则直接返回原 df，不做任何调整。
+    """
+    # 直接返回原 df 的情况
+    if set_type is None:
+        return df
+    if isinstance(set_type, Mapping) and len(set_type) == 0:
+        return df
+    if isinstance(set_type, Sequence) and not isinstance(set_type, str) and len(set_type) == 0:
+        return df
+
+    # 如果set_type是字典，提取其键名
+    if isinstance(set_type, Mapping):
+        col_order = list(set_type.keys())
+    elif isinstance(set_type, Sequence) and not isinstance(set_type, str):
+        col_order = list(set_type)
+    else:
+        raise ValueError("set_type must be a list or a dict (or other mapping type)")
+
+    # 构建原始列名的映射（小写->原始名）
+    col_map = {col.lower(): col for col in df.columns}
+    # 生成新顺序的列名（只保留df中存在的列，且顺序按set_type）
+    new_cols = []
+    used = set()
+    for col in col_order:
+        key = col.lower()
+        if key in col_map and key not in used:
+            new_cols.append(col_map[key])
+            used.add(key)
+    # 添加剩余未在set_type中出现的列，保持原顺序
+    for col in df.columns:
+        if col.lower() not in used:
+            new_cols.append(col)
+    # 返回新顺序的DataFrame
+    return df[new_cols]
+
+
 def upload_data_decorator(**upload_kwargs):
     """
     数据上传装饰器
@@ -46,6 +88,24 @@ def upload_data_decorator(**upload_kwargs):
         @wraps(func)
         def wrapper(*args, **kwargs):
             try:
+                # 获取 set_type 或 set_typ 参数
+                set_type = None
+                # 先从kwargs查找
+                for key in ['set_type', 'set_typ']:
+                    if key in kwargs:
+                        set_type = kwargs[key]
+                        break
+                # 如果没在kwargs找到，尝试从args按参数名顺序查找
+                if set_type is None:
+                    sig = inspect.signature(func)
+                    params = list(sig.parameters)
+                    for key in ['set_type', 'set_typ']:
+                        if key in params:
+                            idx = params.index(key)
+                            if len(args) > idx:
+                                set_type = args[idx]
+                                break
+
                 # 执行原始函数
                 result = func(*args, **kwargs)
                 
@@ -55,6 +115,9 @@ def upload_data_decorator(**upload_kwargs):
                 
                 # 如果返回的是 DataFrame
                 if isinstance(result, pd.DataFrame):
+                    # 调整列顺序
+                    if set_type is not None:
+                        result = reorder_columns(result, set_type)
                     # 设置默认值
                     default_kwargs = {
                         'check_duplicate': False,
@@ -87,6 +150,11 @@ def upload_data_decorator(**upload_kwargs):
                         logger.warning('函数返回的元组第一个元素不是DataFrame，直接返回原结果，不执行上传', {'函数': func.__name__})
                         return result
                         
+                    # 调整列顺序
+                    if set_type is not None:
+                        df = reorder_columns(df, set_type)
+                        # 保持元组结构
+                        result = (df, extra_kwargs) + result[2:]
                     # 合并装饰器参数和函数参数
                     merged_kwargs = {**upload_kwargs}
                     merged_kwargs.update(extra_kwargs)
@@ -2119,7 +2187,7 @@ class MysqlDatasQuery:
             'partition_date_column': '日期',  # 用于分表的日期列名，默认为'日期'
             'indexes': [],  # 普通索引列
             'transaction_mode': 'batch',  # 事务模式
-            'unique_keys': [['日期', '产品线', '搜索词', '计划id', '搜索词', '关键词']],  # 唯一约束列表
+            'unique_keys': [['日期', '产品线', '计划id', '搜索词', '关键词']],  # 唯一约束列表
         }
 
     @try_except
@@ -3032,7 +3100,7 @@ class MysqlDatasQuery:
             'partition_date_column': '日期',  # 用于分表的日期列名，默认为'日期'
             'indexes': [],  # 普通索引列
             'transaction_mode': 'batch',  # 事务模式
-            'unique_keys': [['日期', '店铺名称', '营销场景']],  # 唯一约束列表
+            'unique_keys': [['日期', '店铺名称', '商品款号', 'spuid']],  # 唯一约束列表
         }
 
     @upload_data_decorator()
@@ -3661,7 +3729,6 @@ def date_table():
     df = df.reset_index(drop=False)
     df.rename(columns={'index': 'id'}, inplace=True)
     df['id'] = df['id'].apply(lambda x: x + 1)
-
     set_typ = {
         '日期': 'date',
         '年': 'varchar(50)',
@@ -3676,6 +3743,7 @@ def date_table():
         '索引': 'int',
         '月索引': 'int',
     }
+
     return df, {
         'db_name': '聚合数据',
         'table_name': '日期表',
@@ -3772,9 +3840,6 @@ def main(days=150, months=3):
 
 
 if __name__ == '__main__':
-    main(
-        days=150,  # 清理聚合数据的日期长度
-        months=3  # 生成聚合数据的长度
-    )
+    # main(months=3)
 
-    # date_table()
+    pass

mdbq/mysql/s_query.py

@@ -762,6 +762,21 @@ class QueryDatas:
             finally:
                 self.pool = None
 
+    def _adjust_page_size(self, last_duration, current_page_size, min_size=1000, max_size=10000, target_time=2.0):
+        """
+        根据上一次批次耗时自动调整下一次的 page_size。
+        - last_duration: 上一批次查询耗时（秒）
+        - current_page_size: 当前批次大小
+        - min_size, max_size: 允许的最小/最大批次
+        - target_time: 期望每批耗时（秒）
+        """
+        if last_duration < target_time / 2 and current_page_size < max_size:
+            return min(current_page_size * 2, max_size)
+        elif last_duration > target_time * 2 and current_page_size > min_size:
+            return max(current_page_size // 2, min_size)
+        else:
+            return current_page_size
+
     def data_to_df(
             self, 
             db_name: str, 
@@ -890,18 +905,20 @@ class QueryDatas:
                     # 分页查询
                     offset = 0
                     all_results = []
-                    
+                    min_size, max_size = 1000, 10000
+                    target_time = 1.0  # 期望每批1秒
+
                     while offset < total_count:
+                        start_time = time.time()
                         # 添加分页参数
                         page_sql = f"{base_sql} LIMIT %s OFFSET %s"
                         page_params = list(params) + [page_size, offset]
-                        
                         cursor.execute(page_sql, tuple(page_params))
                         page_results = cursor.fetchall()
-                        
+
                         if not page_results:
                             break
-                            
+
                         if return_format == 'list_dict':
                             all_results.extend(page_results)
                         else:
@@ -909,14 +926,18 @@ class QueryDatas:
                                 all_results = pd.DataFrame(page_results)
                             else:
                                 all_results = pd.concat([all_results, pd.DataFrame(page_results)], ignore_index=True)
-                        
-                        offset += page_size
+
+                        duration = time.time() - start_time
+                        page_size = self._adjust_page_size(duration, page_size, min_size, max_size, target_time)
+                        offset += len(page_results)
                         logger.debug('分页查询进度', {
                             '库': db_name,
                             '表': table_name,
-                            '当前偏移量': offset,
+                            # '当前偏移量': offset,
                             '总记录数': total_count,
-                            '已获取记录数': len(all_results) if return_format == 'list_dict' else len(all_results.index)
+                            '已获取记录数': len(all_results) if return_format == 'list_dict' else len(all_results.index),
+                            '本批耗时': f'{duration:.2f}',
+                            '下批page_size': page_size
                         })
 
                     if return_format == 'df' and isinstance(all_results, pd.DataFrame) and not all_results.empty:

mdbq/mysql/uploader.py

@@ -14,6 +14,7 @@ from dbutils.pooled_db import PooledDB
 import json
 import sys
 from decimal import Decimal, InvalidOperation
+import math
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
@@ -240,8 +241,16 @@ class MySQLUploader:
             conn = self.pool.connection()
             return conn
         except Exception as e:
-            logger.error('从连接池获取数据库连接失败', {'error': str(e)})
-            raise ConnectionError(f'连接数据库失败: {str(e)}')
+            logger.error('从连接池获取数据库连接失败，尝试重建连接池', {'error': str(e)})
+            # 强制重建连接池
+            try:
+                self.pool = self._create_connection_pool()
+                conn = self.pool.connection()
+                logger.info('重建连接池后获取连接成功')
+                return conn
+            except Exception as e2:
+                logger.error('重建连接池后依然获取连接失败', {'error': str(e2)})
+                raise ConnectionError(f'连接数据库失败: {str(e2)}')
 
     @_execute_with_retry
     def _check_database_exists(self, db_name: str) -> bool:
@@ -407,31 +416,36 @@ class MySQLUploader:
                 col_def += " NOT NULL"
             column_defs.append(col_def)
         # 主键处理逻辑调整
+        def _index_col_sql(col):
+            col_type = set_typ.get(col, '').lower()
+            if 'varchar' in col_type or 'text' in col_type:
+                return f"`{self._normalize_col(col)}`(100)"
+            return f"`{self._normalize_col(col)}`"
         if primary_keys and len(primary_keys) > 0:
-            safe_primary_keys = [self._normalize_col(pk) for pk in primary_keys]
-            primary_key_sql = f"PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+            safe_primary_keys = [_index_col_sql(pk) for pk in primary_keys]
+            primary_key_sql = f"PRIMARY KEY ({','.join(safe_primary_keys)})"
         else:
-            safe_primary_keys = [self._normalize_col('id')]
+            safe_primary_keys = [_index_col_sql('id')]
             primary_key_sql = f"PRIMARY KEY (`id`)"
         # 索引统一在CREATE TABLE中定义
         index_defs = []
         if date_column and date_column in set_typ:
-            safe_date_col = self._normalize_col(date_column)
-            index_defs.append(f"INDEX `idx_{safe_date_col}` (`{safe_date_col}`)")
+            safe_date_col = _index_col_sql(date_column)
+            index_defs.append(f"INDEX `idx_{self._normalize_col(date_column)}` ({safe_date_col})")
         if indexes:
             for idx_col in indexes:
                 if idx_col in set_typ:
-                    safe_idx_col = self._normalize_col(idx_col)
-                    index_defs.append(f"INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)")
+                    safe_idx_col = _index_col_sql(idx_col)
+                    index_defs.append(f"INDEX `idx_{self._normalize_col(idx_col)}` ({safe_idx_col})")
         # UNIQUE KEY定义
         unique_defs = []
         if unique_keys:
             for unique_cols in unique_keys:
                 if not unique_cols:
                     continue
-                safe_unique_cols = [self._normalize_col(col) for col in unique_cols]
-                unique_name = f"uniq_{'_'.join(safe_unique_cols)}"
-                unique_defs.append(f"UNIQUE KEY `{unique_name}` (`{'`,`'.join(safe_unique_cols)}`)")
+                safe_unique_cols = [_index_col_sql(col) for col in unique_cols]
+                unique_name = f"uniq_{'_'.join([self._normalize_col(c) for c in unique_cols])}"
+                unique_defs.append(f"UNIQUE KEY `{unique_name}` ({','.join(safe_unique_cols)})")
         index_defs = list(set(index_defs))
         all_defs = column_defs + [primary_key_sql] + index_defs + unique_defs
         sql = f"""
@@ -447,7 +461,7 @@ class MySQLUploader:
                 conn.commit()
                 logger.info('数据表及索引已创建', {'库': db_name, '表': table_name, '索引': indexes, '唯一约束': unique_keys})
         except Exception as e:
-            logger.error('建表失败', {'库': db_name, '表': table_name, '错误': str(e)})
+            logger.error('建表失败', {'库': db_name, '表': table_name, '错误': str(e), '异常类型': type(e).__name__})
             if conn is not None:
                 conn.rollback()
             raise
@@ -491,25 +505,45 @@ class MySQLUploader:
     def _validate_value(self, value: Any, column_type: str, allow_null: bool, db_name: str = None, table_name: str = None, col_name: str = None) -> Any:
         """
         根据列类型验证并转换数据值
-
-        :param value: 要验证的值
-        :param column_type: 列的数据类型
-        :param allow_null: 是否允许空值
-        :param db_name: 数据库名（用于日志）
-        :param table_name: 表名（用于日志）
-        :param col_name: 列名（用于日志）
-        :return: 转换后的值
-        :raises ValueError: 当值转换失败时抛出
         """
+        column_type_lower = column_type.lower() if column_type else ''
+        # 统一判断None/NaN
+        is_nan = False
         if value is None:
+            is_nan = True
+        elif isinstance(value, float) and math.isnan(value):
+            is_nan = True
+        elif str(value).lower() in ['nan', 'none']:
+            is_nan = True
+        if is_nan:
             if not allow_null:
-                logger.warning('字段值为None但不允许空值, 已填充为none', {
-                    '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                })
-                return 'none'
+                if 'int' in column_type_lower:
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                    })
+                    return 0
+                elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0.0', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                    })
+                    return 0.0
+                elif 'date' in column_type_lower or 'time' in column_type_lower:
+                    # 判断是date还是datetime/timestamp
+                    if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
+                        default_date = '2000-01-01 00:00:00'
+                    else:
+                        default_date = '2000-01-01'
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为默认日期', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type, '默认值': default_date
+                    })
+                    return default_date
+                else:
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为none字符串', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                    })
+                    return 'none'
             return None
         try:
-            column_type_lower = column_type.lower()
             if isinstance(value, str) and value.strip().endswith('%'):
                 try:
                     percent_str = value.strip().replace('%', '')
@@ -881,22 +915,21 @@ class MySQLUploader:
         # set_typ的键清洗
         set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
 
-        # 获取数据中实际存在的列名
-        data_columns = set()
-        if data and len(data) > 0:
-            data_columns = set(data[0].keys())
-
-        # 过滤set_typ，只保留数据中存在的列
+        # 新实现：严格按set_typ顺序过滤，后补充data中有但set_typ没有的列
         filtered_set_typ = {}
-        for col in data_columns:
-            if col in set_typ:
+        data_columns = list(data[0].keys()) if data and len(data) > 0 else []
+        # 先按set_typ顺序
+        for col in set_typ:
+            if col in data_columns:
                 filtered_set_typ[col] = set_typ[col]
-            else:
-                # 如果列不在set_typ中，采样多个非None值推断类型
+        # 再补充data中有但set_typ没有的列
+        for col in data_columns:
+            if col not in filtered_set_typ:
+                # 推断类型
                 sample_values = [row[col] for row in data if col in row and row[col] is not None][:5]
                 inferred_type = None
                 for val in sample_values:
-                    inferred_type = self._infer_data_type(val, no_log=True)  # 推断日期类型不记录日志, 避免日志噪音过多
+                    inferred_type = self._infer_data_type(val, no_log=True)
                     if inferred_type:
                         break
                 if not inferred_type:
@@ -1326,8 +1359,7 @@ class MySQLUploader:
         if cached:
             return cached
         # 获取所有列名（排除id）
-        all_columns = [col for col in set_typ.keys()
-                       if col.lower() != 'id']
+        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
         if not check_duplicate:
             sql = self._build_simple_insert_sql(db_name, table_name, all_columns,
                                                  update_on_duplicate)
@@ -1364,7 +1396,6 @@ class MySQLUploader:
         - 只有遇到严重的数据库错误（如所有行都因唯一约束冲突且没有ON DUPLICATE KEY UPDATE），才会整体回滚。
         - 返回值为(插入行数, 跳过行数, 失败行数)。
         """
-        import pymysql  # 确保异常类型可用
         def get_optimal_batch_size(total_rows: int) -> int:
             if total_rows <= 100:
                 return total_rows
@@ -1612,5 +1643,5 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    # main()
     pass

{mdbq-4.0.3.dist-info → mdbq-4.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.3
+Version: 4.0.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.3.dist-info → mdbq-4.0.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=TYBD_E4OJ0j_9uluVmIIAD3mWIJUYZHLFUE8yegEN5A,17
+mdbq/__version__.py,sha256=lAYjWBa6ThlPWc3_1b0lWM6fn_3Z9ckuALnMMZXhBbs,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=10qzvBC9s6g2avtyDawUycEMEsadkFM9rkjmIfVLcG8,169012
+mdbq/aggregation/query_data.py,sha256=_k6Jg60RaaT056sIaiSO6v84dEnOIOGq-nUJtSr65kI,171861
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -10,9 +10,9 @@ mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,16
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=hi9qzeMwMyCUG0CCLHBBoQwwbfwOeUnaghJs2iqe9JE,40816
+mdbq/mysql/s_query.py,sha256=RnVCwMQ_n9PcAimbMWbHe9k8eil8shtCfa3LwLBZi6c,41909
 mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=n6K2rn7cGHGLGevX2JO7pKrUiQiGNluRx3CwnxARZGI,72503
+mdbq/mysql/uploader.py,sha256=e49Gk09K766QXaus_p3VOMcH2VbexQzKsqDTCGrWoWQ,74419
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=l3zBK7wrZl0oO42-_UGylyatnIp_SBw8wDDvof9fht4,23534
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.3.dist-info/METADATA,sha256=Iy_UiPHt9ljSMXDbNVzkekDBZBEpWM-NCRR5VQLfq-g,363
-mdbq-4.0.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.3.dist-info/RECORD,,
+mdbq-4.0.4.dist-info/METADATA,sha256=VM2dtOiBJ74NlYhq9UWAFAPloayAXPX9bLKnvZJd7Xg,363
+mdbq-4.0.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.4.dist-info/RECORD,,