mdbq 4.0.39__py3-none-any.whl → 4.0.41__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.39'
+VERSION = '4.0.41'

mdbq/mysql/uploader.py

@@ -11,7 +11,6 @@ from mdbq.log import mylogger
 from mdbq.myconf import myconf
 from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
-import json
 import sys
 from decimal import Decimal, InvalidOperation
 import math
@@ -526,8 +525,7 @@ class MySQLUploader:
             '%Y-%m-%dT%H:%M:%S',
             '%Y-%m-%d %H:%M:%S.%f',
             '%Y/%-m/%-d',  # 2023/1/8
-            '%Y-%m-%-d',  # 2023-01-8
-            '%Y-%-m-%-d'  # 2023-1-8
+            '%Y-%-m-%-d',  # 2023-01-8
         ]
         for fmt in formats:
             try:
@@ -549,106 +547,70 @@ class MySQLUploader:
         """
         column_type_lower = column_type.lower() if column_type else ''
         # 统一判断None/NaN
-        is_nan = False
-        if value is None:
-            is_nan = True
-        elif isinstance(value, float) and math.isnan(value):
-            is_nan = True
-        elif str(value).lower() in ['nan', 'none']:
-            is_nan = True
-        elif value == '':
-            is_nan = True
-        if is_nan:
-            if not allow_null:
-                if 'int' in column_type_lower:
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                    })
-                    return 0
-                elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0.0', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                    })
-                    return 0.0
-                elif 'date' in column_type_lower or 'time' in column_type_lower:
-                    if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
-                        default_date = '2000-01-01 00:00:00'
-                    else:
-                        default_date = '2000-01-01'
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为默认日期', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type, '默认值': default_date
-                    })
-                    return default_date
-                else:
-                    logger.debug('字段值为None/NaN但不允许空值, 已填充为none字符串', {
-                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                    })
-                    return 'none'
-            return None
-        try:
-            if isinstance(value, str) and value.strip().endswith('%'):
-                if re.match(r'^\d+(\.\d+)?%$', value.strip()):
-                    percent_str = value.strip().replace('%', '')
-                    percent_value = float(percent_str)
-                    decimal_value = percent_value / 100
-                    logger.debug('百分比字符串转小数', {'原始': value, '结果': decimal_value})
-                    return decimal_value
-                else:
-                    logger.warning('百分比字符串不符合格式，跳过转换', {
-                        '库': db_name, '表': table_name, '列': col_name, '原始': value
-                    })
-            elif 'int' in column_type_lower:
-                if isinstance(value, str):
-                    value = value.replace(',', '').strip()
-                    try:
-                        return int(float(value))
-                    except ValueError:
-                        logger.error('字符串转整数失败', {
-                            '库': db_name, '表': table_name, '列': col_name, '值': value
-                        })
-                        raise ValueError(f"`{value}` -> 无法转为整数")
-                return int(value) if value is not None else None
-            elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
-                if isinstance(value, str):
-                    value = value.replace(',', '')
-                return float(value) if value is not None else None
-            elif 'date' in column_type_lower or 'time' in column_type_lower:
-                if isinstance(value, (datetime.datetime, pd.Timestamp)):
-                    return value.strftime('%Y-%m-%d %H:%M:%S')
-                elif isinstance(value, str):
-                    try:
-                        return self._validate_datetime(value=value, date_type=False, no_log=False)
-                    except ValueError as e:
-                        logger.error('无效日期格式', {
-                            '库': db_name, '表': table_name, '列': col_name, '值': value, '错误': str(e)
-                        })
-                        raise ValueError(f"无效日期格式: `{value}` -> {str(e)}")
-                return str(value)
-            elif 'varchar' in column_type_lower:
-                if isinstance(value, str):
-                    return value.replace('\\', '\\\\').replace("'", "\\'")
-                else:
-                    return str(value)
-            elif 'text' in column_type_lower:
-                if isinstance(value, str):
-                    max_length = 65535
-                    if len(value) > max_length:
-                        logger.warning(f'TEXT字符串长度不允许超过 {max_length}，已截断', {
-                            '库': db_name, '表': table_name, '列': col_name, '原始值': f'{value[:50]}...', '截断后值': f'{value[:50]}...'
-                        })
-                        value = value[:max_length]
-                    return value.replace('\\', '\\\\').replace("'", "\\'")
-                else:
-                    return str(value)
-            elif 'json' in column_type_lower:
-                return json.dumps(value) if value is not None else None
+        if value == '' or pd.isna(value) or (isinstance(value, (float, Decimal)) and math.isinf(value)):
+            if allow_null:
+                return None
             else:
-                return value
-        except (ValueError, TypeError) as e:
-            logger.error('数据类型转换异常', {
-                '库': db_name, '表': table_name, '列': col_name, '值': value, '目标类型': column_type, '错误': str(e)
-            })
-            raise ValueError(f"转换异常 -> 无法将 `{value}` 的数据类型转为: `{column_type}` -> {str(e)}")
+                logger.error("该列不允许为空值", {"库": db_name, "表": table_name, "列": col_name, "值": value})
+                raise ValueError("该列不允许为空值")
+
+        original_value = value
+
+        # 日期时间类型验证
+        if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
+            return self._validate_datetime(value, date_type=False, no_log=True)
+        elif 'date' in column_type_lower:
+            return self._validate_datetime(value, date_type=True, no_log=True)
+        # 数值类型验证
+        elif 'int' in column_type_lower:
+            try:
+                return int(value)
+            except (ValueError, TypeError):
+                logger.error(f"值 `{value}` 无法转换为整数", {"库": db_name, "表": table_name, "列": col_name})
+                raise ValueError(f"值 `{value}` 无法转换为整数")
+        elif any(t in column_type_lower for t in ['decimal', 'float', 'double']):
+            # 百分比字符串处理
+            if isinstance(value, str) and '%' in value:
+                try:
+                    # 仅当值是'xx.xx%'格式时才转换
+                    if re.match(r'^-?\d+(\.\d+)?%$', value.strip()):
+                        value = float(value.strip().replace('%', '')) / 100
+                    else:
+                        # 不符合格式的百分比字符串，保留原始值
+                        logger.warning("百分比字符串不符合格式，跳过转换", {"库": db_name, "表": table_name, "列": col_name, "原始": original_value})
+                        value = original_value
+                except (ValueError, TypeError):
+                    logger.warning("百分比字符串转换失败，保留原始值", {"库": db_name, "表": table_name, "列": col_name, "原始": original_value})
+                    value = original_value
+
+            try:
+                # 检查是否为Decimal类型
+                if 'decimal' in column_type_lower:
+                    precision, scale = self._get_decimal_scale(column_type)
+                    value_decimal = Decimal(str(value))
+                    # 检查整数部分长度
+                    if len(value_decimal.as_tuple().digits) - abs(value_decimal.as_tuple().exponent) > precision - scale:
+                        raise ValueError(f"整数部分超出范围")
+                    return value_decimal
+                else: # float/double
+                    return float(value)
+            except (ValueError, TypeError, InvalidOperation) as e:
+                logger.error(f"值 `{value}` 无法转换为数值类型: {e}", {"库": db_name, "表": table_name, "列": col_name})
+                raise ValueError(f"值 `{value}` 无法转换为数值类型: {e}")
+        # 字符串类型验证
+        elif 'varchar' in column_type_lower:
+            str_value = str(value)
+            try:
+                max_len = int(re.search(r'\((\d+)\)', column_type).group(1))
+                if len(str_value.encode('utf-8')) > max_len:
+                    logger.warning(f"列`{col_name}`的值`{str_value}`长度({len(str_value.encode('utf-8'))})超出varchar({max_len})限制，将进行截断", {"库": db_name, "表": table_name})
+                    return str_value.encode('utf-8')[:max_len].decode('utf-8', 'ignore')
+            except (AttributeError, IndexError):
+                # 没有找到长度定义，不截断
+                pass
+            return str_value
+        
+        return value
 
     @_execute_with_retry
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
@@ -853,8 +815,11 @@ class MySQLUploader:
 
         # 检查是否是百分比字符串
         if isinstance(value, str):
-            if value.endswith('%'):
-                return 'DECIMAL(10,4)'  # 百分比统一使用DECIMAL(10,4)
+            if '%' in value:
+                if re.match(r'^-?\d+(\.\d+)?%$', value.strip()):
+                    return 'DECIMAL(10, 4)'  # 百分比转为小数，使用DECIMAL
+                else:
+                    return 'VARCHAR(255)' # 不符合格式的百分比，视为字符串
 
         if isinstance(value, bool):
             return 'TINYINT(1)'
@@ -898,8 +863,8 @@ class MySQLUploader:
                 return 'MEDIUMTEXT'
             else:
                 return 'LONGTEXT'
-        else:
-            return 'VARCHAR(255)'
+            
+        return 'VARCHAR(255)' # 默认字符串类型
 
     def normalize_column_names(self, data: Union[pd.DataFrame, List[Dict[str, Any]]]) -> Union[
         pd.DataFrame, List[Dict[str, Any]]]:
@@ -967,6 +932,8 @@ class MySQLUploader:
         data = self.normalize_column_names(data)
 
         # set_typ的键清洗
+        if not set_typ:
+            set_typ = {}
         set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
 
         # 新实现：严格按set_typ顺序过滤，后补充data中有但set_typ没有的列
@@ -1017,7 +984,6 @@ class MySQLUploader:
                         })
                         raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
             prepared_data.append(prepared_row)
-
         return prepared_data, filtered_set_typ
 
     def upload_data(
@@ -1353,7 +1319,7 @@ class MySQLUploader:
         for col in dup_cols:
             col_type = set_typ.get(col, '').lower()
             if col_type.startswith('decimal'):
-                scale = self._get_decimal_scale(col_type)
+                _, scale = self._get_decimal_scale(col_type)
                 conditions.append(f"ROUND(`{col}`, {scale}) = ROUND(%s, {scale})")
             else:
                 conditions.append(f"`{col}` = %s")
@@ -1382,24 +1348,12 @@ class MySQLUploader:
             """
         return sql
 
-    def _get_decimal_scale(self, decimal_type: str) -> int:
-        """
-        从DECIMAL类型定义中提取小数位数
-
-        :param decimal_type: DECIMAL类型字符串，如'DECIMAL(10,4)'
-        :return: 小数位数
-        :raises: 无显式抛出异常，但解析失败时返回默认值2
-        """
-        try:
-            # 匹配DECIMAL类型中的精度和小数位数
-            match = re.match(r'decimal\((\d+),\s*(\d+)\)', decimal_type.lower())
-            if match:
-                return int(match.group(2))
-        except (ValueError, AttributeError, IndexError):
-            pass
-
-        # 默认返回2位小数
-        return 2
+    def _get_decimal_scale(self, decimal_type: str) -> Tuple[int, int]:
+        """从DECIMAL类型字符串中提取精度和标度"""
+        match = re.search(r'\((\d+)\s*,\s*(\d+)\)', decimal_type)
+        if match:
+            return int(match.group(1)), int(match.group(2))
+        return 18, 2  # 默认值
 
     def _prepare_insert_sql(
             self,

{mdbq-4.0.39.dist-info → mdbq-4.0.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.39
+Version: 4.0.41
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.39.dist-info → mdbq-4.0.41.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=n_fdVNnrFSkcZj0OrtGWK_KfQ2bTndWqHA32sOOig7o,18
+mdbq/__version__.py,sha256=X4VgvOxMoRU-YHIymW2NhCV8yy6pvwj9wlFGdVP8Di8,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=WtTFMN78jn43Y-nBTPAXhAK56w3wDuv_cj4YtzzGbZk,169797
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -11,7 +11,7 @@ mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=1wJ3HVjHEF6FA-bVeeesRlsf73CZSvVTEQ51CF1OsE4,46786
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=defQ4xCC3j8an9dWjFI3q_Fec0Irewe2FzBZqFL1GJM,84673
+mdbq/mysql/uploader.py,sha256=PQKFohU32hRyUW3sAgWigbLnBh4h9ZydF4RNC3wNAyo,82640
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=XptHjGzbout9IYzWAOQUpMMV5qEgLTU8pL1ZGt8oNEA,21868
-mdbq-4.0.39.dist-info/METADATA,sha256=9JtfqN1rvyf6K99crg26QGu--t9ksGomocg1VPyMHp8,364
-mdbq-4.0.39.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.39.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.39.dist-info/RECORD,,
+mdbq-4.0.41.dist-info/METADATA,sha256=FDyMAo_9iwFqvELSDiPqPM6GhkNj0htFROeZiZYNMcU,364
+mdbq-4.0.41.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.41.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.41.dist-info/RECORD,,