mdbq 4.0.4__py3-none-any.whl → 4.0.5__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.4'
+VERSION = '4.0.5'

mdbq/aggregation/query_data.py

@@ -87,108 +87,90 @@ def upload_data_decorator(**upload_kwargs):
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
+            db_name = None
+            table_name = None
             try:
-                # 获取 set_type 或 set_typ 参数
-                set_type = None
-                # 先从kwargs查找
-                for key in ['set_type', 'set_typ']:
-                    if key in kwargs:
-                        set_type = kwargs[key]
-                        break
-                # 如果没在kwargs找到，尝试从args按参数名顺序查找
-                if set_type is None:
-                    sig = inspect.signature(func)
-                    params = list(sig.parameters)
-                    for key in ['set_type', 'set_typ']:
-                        if key in params:
-                            idx = params.index(key)
-                            if len(args) > idx:
-                                set_type = args[idx]
-                                break
+                # 获取函数签名和参数
+                sig = inspect.signature(func)
+                bound_args = sig.bind(*args, **kwargs)
+                args_dict = bound_args.arguments
+
+                # 获取所需参数
+                def get_param_value(param_name, alternatives=None):
+                    if alternatives is None:
+                        alternatives = [param_name]
+                    # 从 kwargs 或 args_dict 中获取参数值
+                    for key in alternatives:
+                        if key in kwargs:
+                            return kwargs[key]
+                        if key in args_dict:
+                            return args_dict[key]
+                    return None
+
+                # 获取参数值
+                set_type = get_param_value('set_type', ['set_type', 'set_typ'])
+                db_name = get_param_value('db_name')
+                table_name = get_param_value('table_name')
 
                 # 执行原始函数
                 result = func(*args, **kwargs)
                 
-                # 如果返回 None，直接返回
                 if result is None:
                     return None
-                
-                # 如果返回的是 DataFrame
+
+                # 处理 DataFrame 结果
                 if isinstance(result, pd.DataFrame):
-                    # 调整列顺序
                     if set_type is not None:
                         result = reorder_columns(result, set_type)
-                    # 设置默认值
-                    default_kwargs = {
+                    
+                    # 合并参数
+                    merged_kwargs = {
                         'check_duplicate': False,
                         'update_on_duplicate': True,
                         'allow_null': False,
-                        'transaction_mode': 'batch'
+                        'transaction_mode': 'batch',
+                        **upload_kwargs
                     }
-                    # 更新参数，优先使用装饰器参数
-                    merged_kwargs = {**default_kwargs, **upload_kwargs}
                     
-                    # 上传数据
-                    uld.upload_data(
-                        data=result,
-                        **merged_kwargs
-                    )
+                    uld.upload_data(data=result, **merged_kwargs)
                     return True
-                
-                # 如果返回的是元组
+
+                # 处理元组结果
                 elif isinstance(result, tuple):
-                    # 检查元组长度
                     if len(result) < 2:
-                        logger.warning('函数返回的元组长度小于2，直接返回原结果，不执行上传', {'函数': func.__name__})
+                        logger.warning('函数返回的元组长度小于2，直接返回原结果，不执行上传', {'函数': func.__name__, '库': db_name, '表': table_name})
                         return result
-                        
-                    # 获取前两个元素
+
                     df, extra_kwargs = result[0], result[1]
                     
-                    # 检查第一个元素是否为DataFrame
                     if not isinstance(df, pd.DataFrame):
-                        logger.warning('函数返回的元组第一个元素不是DataFrame，直接返回原结果，不执行上传', {'函数': func.__name__})
+                        logger.warning('函数返回的元组第一个元素不是DataFrame，直接返回原结果，不执行上传', {'函数': func.__name__, '库': db_name, '表': table_name})
                         return result
-                        
-                    # 调整列顺序
+
                     if set_type is not None:
                         df = reorder_columns(df, set_type)
-                        # 保持元组结构
                         result = (df, extra_kwargs) + result[2:]
-                    # 合并装饰器参数和函数参数
-                    merged_kwargs = {**upload_kwargs}
-                    merged_kwargs.update(extra_kwargs)
-                    
-                    # 设置默认值
-                    default_kwargs = {
+
+                    # 合并参数
+                    merged_kwargs = {
                         'check_duplicate': False,
                         'update_on_duplicate': True,
                         'allow_null': False,
-                        'transaction_mode': 'batch'
+                        'transaction_mode': 'batch',
+                        **upload_kwargs,
+                        **extra_kwargs
                     }
-                    # 更新参数，优先使用装饰器参数
-                    for key, value in default_kwargs.items():
-                        if key not in merged_kwargs:
-                            merged_kwargs[key] = value
                     
-                    # 上传数据
-                    uld.upload_data(
-                        data=df,
-                        **merged_kwargs
-                    )
+                    uld.upload_data(data=df, **merged_kwargs)
                     
-                    # 如果元组长度大于2，返回完整元组
-                    if len(result) > 2:
-                        return result
-                    return True
-                
-                # 其他情况直接返回结果
+                    return result if len(result) > 2 else True
+
                 return result
-                
+
             except Exception as e:
-                logger.error('数据上传失败', {'函数': func.__name__, '错误': str(e)})
+                logger.error('数据上传失败', {'函数': func.__name__, '库': db_name, '表': table_name, '错误': str(e)})
                 return False
-                
+
         return wrapper
     return decorator
 
@@ -2019,7 +2001,7 @@ class MysqlDatasQuery:
             'partition_date_column': '日期',  # 用于分表的日期列名，默认为'日期'
             'indexes': [],  # 普通索引列
             'transaction_mode': 'batch',  # 事务模式
-            'unique_keys': [['日期', '店铺名称', '产品线', '触发sku_id', '跟单sku_id']],  # 唯一约束列表
+            'unique_keys': [['日期', '店铺名称', '产品线', '触发sku_id', '跟单sku_id', '花费']],  # 唯一约束列表
         }
 
     @try_except
@@ -3761,12 +3743,7 @@ def date_table():
     }
 
 
-def query1(months=1, less_dict=None, download_manager=None):
-    if less_dict is None:
-        less_dict = []
-    if months == 0:
-        logger.info('months 不建议为 0')
-        return
+def query1(months=1, download_manager=None):
     sdq = MysqlDatasQuery(download_manager=download_manager)  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
 
@@ -3793,12 +3770,7 @@ def query1(months=1, less_dict=None, download_manager=None):
     sdq.performance_concat(bb_tg=False, db_name='聚合数据', table_name='天猫_推广汇总')  # _推广商品销售
 
 
-def query2(months=1, less_dict=None, download_manager=None):
-    if less_dict is None:
-        less_dict = []
-    if months == 0:
-        logger.info('months 不建议为 0')
-        return
+def query2(months=1, download_manager=None):
     sdq = MysqlDatasQuery(download_manager=download_manager)  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
     sdq.dplyd(db_name='聚合数据', table_name='店铺流量来源构成')
@@ -3811,18 +3783,13 @@ def query2(months=1, less_dict=None, download_manager=None):
     sdq.deeplink(db_name='聚合数据', table_name='达摩盘_deeplink人群洞察')
 
 
-def query3(months=1, less_dict=None, download_manager=None):
-    if less_dict is None:
-        less_dict = []
-    if months == 0:
-        logger.info('months 不建议为 0')
-        return
+def query3(months=1, download_manager=None):
     sdq = MysqlDatasQuery(download_manager=download_manager)  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
     sdq.spph(db_name='聚合数据', table_name='天猫_商品排行')
 
 
-def main(days=150, months=3):
+def main(months=3):
     # 1. 更新日期表  更新货品年份基准表， 属性设置 3 - 货品年份基准
     date_table()
 

mdbq/mysql/uploader.py

@@ -528,7 +528,6 @@ class MySQLUploader:
                     })
                     return 0.0
                 elif 'date' in column_type_lower or 'time' in column_type_lower:
-                    # 判断是date还是datetime/timestamp
                     if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
                         default_date = '2000-01-01 00:00:00'
                     else:
@@ -545,14 +544,14 @@ class MySQLUploader:
             return None
         try:
             if isinstance(value, str) and value.strip().endswith('%'):
-                try:
+                if re.match(r'^\d+(\.\d+)?%$', value.strip()):
                     percent_str = value.strip().replace('%', '')
                     percent_value = float(percent_str)
                     decimal_value = percent_value / 100
                     logger.debug('百分比字符串转小数', {'原始': value, '结果': decimal_value})
                     return decimal_value
-                except ValueError:
-                    logger.warning('百分比字符串转小数失败', {
+                else:
+                    logger.warning('百分比字符串不符合格式，跳过转换', {
                         '库': db_name, '表': table_name, '列': col_name, '原始': value
                     })
             elif 'int' in column_type_lower:
@@ -582,10 +581,18 @@ class MySQLUploader:
                         })
                         raise ValueError(f"无效日期格式: `{value}` -> {str(e)}")
                 return str(value)
-            elif 'char' in column_type_lower or 'text' in column_type_lower:
+            elif 'varchar' in column_type_lower:
                 if isinstance(value, str):
                     return value.replace('\\', '\\\\').replace("'", "\\'")
-                return str(value)
+            elif 'text' in column_type_lower:
+                if isinstance(value, str):
+                    max_length = 65535
+                    if len(value) > max_length:
+                        logger.warning(f'TEXT字符串长度不允许超过 {max_length}，已截断', {
+                            '库': db_name, '表': table_name, '列': col_name, '原始值': f'{value[:50]}...', '截断后值': f'{value[:50]}...'
+                        })
+                        value = value[:max_length]
+                return value.replace('\\', '\\\\').replace("'", "\\'")
             elif 'json' in column_type_lower:
                 return json.dumps(value) if value is not None else None
             else:

{mdbq-4.0.4.dist-info → mdbq-4.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.4
+Version: 4.0.5
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.4.dist-info → mdbq-4.0.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=lAYjWBa6ThlPWc3_1b0lWM6fn_3Z9ckuALnMMZXhBbs,17
+mdbq/__version__.py,sha256=K0JdqT-aY_eW77ySyyxnpc599EoZ9CKOLZg_w5AvAnM,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=_k6Jg60RaaT056sIaiSO6v84dEnOIOGq-nUJtSr65kI,171861
+mdbq/aggregation/query_data.py,sha256=3GBdX0HWKvQ-B3NiZE_hzWbJ7sqClzCd8KTvXpVPnZ4,170452
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -12,7 +12,7 @@ mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=RnVCwMQ_n9PcAimbMWbHe9k8eil8shtCfa3LwLBZi6c,41909
 mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=e49Gk09K766QXaus_p3VOMcH2VbexQzKsqDTCGrWoWQ,74419
+mdbq/mysql/uploader.py,sha256=bYE_VGTeEigpRFYvZ9Ob3A9vxq21NuOdrXFkv8Bm_p8,74919
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=l3zBK7wrZl0oO42-_UGylyatnIp_SBw8wDDvof9fht4,23534
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.4.dist-info/METADATA,sha256=VM2dtOiBJ74NlYhq9UWAFAPloayAXPX9bLKnvZJd7Xg,363
-mdbq-4.0.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.4.dist-info/RECORD,,
+mdbq-4.0.5.dist-info/METADATA,sha256=boklJ7iCN4Uh-Czst1DiQlPrKKSawDIYknmipAd9w5A,363
+mdbq-4.0.5.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.5.dist-info/RECORD,,