mdbq 4.0.39__py3-none-any.whl → 4.0.40__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.39'
+VERSION = '4.0.40'

mdbq/mysql/uploader.py

@@ -11,7 +11,6 @@ from mdbq.log import mylogger
 from mdbq.myconf import myconf
 from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
-import json
 import sys
 from decimal import Decimal, InvalidOperation
 import math
@@ -549,106 +548,70 @@ class MySQLUploader:
         """
         column_type_lower = column_type.lower() if column_type else ''
         # 统一判断None/NaN
-        is_nan = False
-        if value is None:
-            is_nan = True
-        elif isinstance(value, float) and math.isnan(value):
-            is_nan = True
-        elif str(value).lower() in ['nan', 'none']:
-            is_nan = True
-        elif value == '':
-            is_nan = True
-        if is_nan:
-            if not allow_null:
-                if 'int' in column_type_lower:
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                    })
-                    return 0
-                elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0.0', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                    })
-                    return 0.0
-                elif 'date' in column_type_lower or 'time' in column_type_lower:
-                    if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
-                        default_date = '2000-01-01 00:00:00'
-                    else:
-                        default_date = '2000-01-01'
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为默认日期', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type, '默认值': default_date
-                    })
-                    return default_date
-                else:
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为none字符串', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                    })
-                    return 'none'
-            return None
-        try:
-            if isinstance(value, str) and value.strip().endswith('%'):
-                if re.match(r'^\d+(\.\d+)?%$', value.strip()):
-                    percent_str = value.strip().replace('%', '')
-                    percent_value = float(percent_str)
-                    decimal_value = percent_value / 100
-                    logger.debug('百分比字符串转小数', {'原始': value, '结果': decimal_value})
-                    return decimal_value
-                else:
-                    logger.warning('百分比字符串不符合格式，跳过转换', {
-                        '库': db_name, '表': table_name, '列': col_name, '原始': value
-                    })
-            elif 'int' in column_type_lower:
-                if isinstance(value, str):
-                    value = value.replace(',', '').strip()
-                    try:
-                        return int(float(value))
-                    except ValueError:
-                        logger.error('字符串转整数失败', {
-                            '库': db_name, '表': table_name, '列': col_name, '值': value
-                        })
-                        raise ValueError(f"`{value}` -> 无法转为整数")
-                return int(value) if value is not None else None
-            elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
-                if isinstance(value, str):
-                    value = value.replace(',', '')
-                return float(value) if value is not None else None
-            elif 'date' in column_type_lower or 'time' in column_type_lower:
-                if isinstance(value, (datetime.datetime, pd.Timestamp)):
-                    return value.strftime('%Y-%m-%d %H:%M:%S')
-                elif isinstance(value, str):
-                    try:
-                        return self._validate_datetime(value=value, date_type=False, no_log=False)
-                    except ValueError as e:
-                        logger.error('无效日期格式', {
-                            '库': db_name, '表': table_name, '列': col_name, '值': value, '错误': str(e)
-                        })
-                        raise ValueError(f"无效日期格式: `{value}` -> {str(e)}")
-                return str(value)
-            elif 'varchar' in column_type_lower:
-                if isinstance(value, str):
-                    return value.replace('\\', '\\\\').replace("'", "\\'")
-                else:
-                    return str(value)
-            elif 'text' in column_type_lower:
-                if isinstance(value, str):
-                    max_length = 65535
-                    if len(value) > max_length:
-                        logger.warning(f'TEXT字符串长度不允许超过 {max_length}，已截断', {
-                            '库': db_name, '表': table_name, '列': col_name, '原始值': f'{value[:50]}...', '截断后值': f'{value[:50]}...'
-                        })
-                        value = value[:max_length]
-                    return value.replace('\\', '\\\\').replace("'", "\\'")
-                else:
-                    return str(value)
-            elif 'json' in column_type_lower:
-                return json.dumps(value) if value is not None else None
+        if value == '' or pd.isna(value) or (isinstance(value, (float, Decimal)) and math.isinf(value)):
+            if allow_null:
+                return None
             else:
-                return value
-        except (ValueError, TypeError) as e:
-            logger.error('数据类型转换异常', {
-                '库': db_name, '表': table_name, '列': col_name, '值': value, '目标类型': column_type, '错误': str(e)
-            })
-            raise ValueError(f"转换异常 -> 无法将 `{value}` 的数据类型转为: `{column_type}` -> {str(e)}")
+                logger.error("该列不允许为空值", {"库": db_name, "表": table_name, "列": col_name, "值": value})
+                raise ValueError("该列不允许为空值")
+
+        original_value = value
+
+        # 日期时间类型验证
+        if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
+            return self._validate_datetime(value, date_type=False, no_log=True)
+        elif 'date' in column_type_lower:
+            return self._validate_datetime(value, date_type=True, no_log=True)
+        # 数值类型验证
+        elif 'int' in column_type_lower:
+            try:
+                return int(value)
+            except (ValueError, TypeError):
+                logger.error(f"值 `{value}` 无法转换为整数", {"库": db_name, "表": table_name, "列": col_name})
+                raise ValueError(f"值 `{value}` 无法转换为整数")
+        elif any(t in column_type_lower for t in ['decimal', 'float', 'double']):
+            # 百分比字符串处理
+            if isinstance(value, str) and '%' in value:
+                try:
+                    # 仅当值是'xx.xx%'格式时才转换
+                    if re.match(r'^-?\d+(\.\d+)?%$', value.strip()):
+                        value = float(value.strip().replace('%', '')) / 100
+                    else:
+                        # 不符合格式的百分比字符串，保留原始值
+                        logger.warning("百分比字符串不符合格式，跳过转换", {"库": db_name, "表": table_name, "列": col_name, "原始": original_value})
+                        value = original_value
+                except (ValueError, TypeError):
+                    logger.warning("百分比字符串转换失败，保留原始值", {"库": db_name, "表": table_name, "列": col_name, "原始": original_value})
+                    value = original_value
+
+            try:
+                # 检查是否为Decimal类型
+                if 'decimal' in column_type_lower:
+                    precision, scale = self._get_decimal_scale(column_type)
+                    value_decimal = Decimal(str(value))
+                    # 检查整数部分长度
+                    if len(value_decimal.as_tuple().digits) - abs(value_decimal.as_tuple().exponent) > precision - scale:
+                        raise ValueError(f"整数部分超出范围")
+                    return value_decimal
+                else: # float/double
+                    return float(value)
+            except (ValueError, TypeError, InvalidOperation) as e:
+                logger.error(f"值 `{value}` 无法转换为数值类型: {e}", {"库": db_name, "表": table_name, "列": col_name})
+                raise ValueError(f"值 `{value}` 无法转换为数值类型: {e}")
+        # 字符串类型验证
+        elif 'varchar' in column_type_lower:
+            str_value = str(value)
+            try:
+                max_len = int(re.search(r'\((\d+)\)', column_type).group(1))
+                if len(str_value.encode('utf-8')) > max_len:
+                    logger.warning(f"列`{col_name}`的值`{str_value}`长度({len(str_value.encode('utf-8'))})超出varchar({max_len})限制，将进行截断", {"库": db_name, "表": table_name})
+                    return str_value.encode('utf-8')[:max_len].decode('utf-8', 'ignore')
+            except (AttributeError, IndexError):
+                # 没有找到长度定义，不截断
+                pass
+            return str_value
+        
+        return value
 
     @_execute_with_retry
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
@@ -967,6 +930,8 @@ class MySQLUploader:
         data = self.normalize_column_names(data)
 
         # set_typ的键清洗
+        if not set_typ:
+            set_typ = {}
         set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
 
         # 新实现：严格按set_typ顺序过滤，后补充data中有但set_typ没有的列
@@ -1017,7 +982,6 @@ class MySQLUploader:
                         })
                         raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
             prepared_data.append(prepared_row)
-
         return prepared_data, filtered_set_typ
 
     def upload_data(

{mdbq-4.0.39.dist-info → mdbq-4.0.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.39
+Version: 4.0.40
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.39.dist-info → mdbq-4.0.40.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=n_fdVNnrFSkcZj0OrtGWK_KfQ2bTndWqHA32sOOig7o,18
+mdbq/__version__.py,sha256=YnJ4yS3LCjrBMdYJ8VXs6BM72GyNYWV1Zwk0iW2hB1k,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=WtTFMN78jn43Y-nBTPAXhAK56w3wDuv_cj4YtzzGbZk,169797
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -11,7 +11,7 @@ mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=1wJ3HVjHEF6FA-bVeeesRlsf73CZSvVTEQ51CF1OsE4,46786
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=defQ4xCC3j8an9dWjFI3q_Fec0Irewe2FzBZqFL1GJM,84673
+mdbq/mysql/uploader.py,sha256=FOroXUIsxJaMCqBeepUuymCpdhJsid4yiC_Rs1BT1sw,82823
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=XptHjGzbout9IYzWAOQUpMMV5qEgLTU8pL1ZGt8oNEA,21868
-mdbq-4.0.39.dist-info/METADATA,sha256=9JtfqN1rvyf6K99crg26QGu--t9ksGomocg1VPyMHp8,364
-mdbq-4.0.39.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.39.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.39.dist-info/RECORD,,
+mdbq-4.0.40.dist-info/METADATA,sha256=FUd8oXQ4zlMFv9neapfYa24uQl84PHgt3S0aAEbEjGo,364
+mdbq-4.0.40.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.40.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.40.dist-info/RECORD,,