mdbq 4.0.40__py3-none-any.whl → 4.0.42__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.40'
+VERSION = '4.0.42'

mdbq/mysql/uploader.py

@@ -516,6 +516,18 @@ class MySQLUploader:
         :return: 标准化后的日期时间字符串或日期对象
         :raises ValueError: 当日期格式无效时抛出
         """
+        # 处理 pandas Timestamp 对象
+        if hasattr(value, 'strftime'):
+            # 如果是 Timestamp 或 datetime 对象，直接格式化
+            if date_type:
+                return pd.to_datetime(value.strftime('%Y-%m-%d'))
+            else:
+                return value.strftime('%Y-%m-%d %H:%M:%S')
+        
+        # 确保 value 是字符串
+        if not isinstance(value, str):
+            value = str(value)
+            
         formats = [
             '%Y-%m-%d %H:%M:%S',
             '%Y-%m-%d',
@@ -525,8 +537,7 @@ class MySQLUploader:
             '%Y-%m-%dT%H:%M:%S',
             '%Y-%m-%d %H:%M:%S.%f',
             '%Y/%-m/%-d',  # 2023/1/8
-            '%Y-%m-%-d',  # 2023-01-8
-            '%Y-%-m-%-d'  # 2023-1-8
+            '%Y-%-m-%-d',  # 2023-01-8
         ]
         for fmt in formats:
             try:
@@ -565,6 +576,24 @@ class MySQLUploader:
         # 数值类型验证
         elif 'int' in column_type_lower:
             try:
+                # 安全地处理各种数值类型
+                if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
+                    # numpy 标量类型
+                    try:
+                        value = value.item()
+                    except (ValueError, TypeError):
+                        # 如果不是标量，保持原值
+                        pass
+                elif hasattr(value, 'value') and not isinstance(value, str):
+                    # pandas 或其他有 value 属性的对象
+                    try:
+                        extracted_value = value.value
+                        # 验证提取的值是数值类型
+                        if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').isdigit():
+                            value = extracted_value
+                    except (ValueError, TypeError, AttributeError):
+                        # 如果提取失败，保持原值
+                        pass
                 return int(value)
             except (ValueError, TypeError):
                 logger.error(f"值 `{value}` 无法转换为整数", {"库": db_name, "表": table_name, "列": col_name})
@@ -588,12 +617,48 @@ class MySQLUploader:
                 # 检查是否为Decimal类型
                 if 'decimal' in column_type_lower:
                     precision, scale = self._get_decimal_scale(column_type)
+                    # 安全地处理各种数值类型
+                    if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
+                        # numpy 标量类型
+                        try:
+                            value = value.item()
+                        except (ValueError, TypeError):
+                            # 如果不是标量，保持原值
+                            pass
+                    elif hasattr(value, 'value') and not isinstance(value, str):
+                        # pandas 或其他有 value 属性的对象
+                        try:
+                            extracted_value = value.value
+                            # 验证提取的值是数值类型
+                            if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').replace('e', '').replace('E', '').isdigit():
+                                value = extracted_value
+                        except (ValueError, TypeError, AttributeError):
+                            # 如果提取失败，保持原值
+                            pass
                     value_decimal = Decimal(str(value))
                     # 检查整数部分长度
                     if len(value_decimal.as_tuple().digits) - abs(value_decimal.as_tuple().exponent) > precision - scale:
                         raise ValueError(f"整数部分超出范围")
                     return value_decimal
                 else: # float/double
+                    # 安全地处理各种数值类型
+                    if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
+                        # numpy 标量类型
+                        try:
+                            value = value.item()
+                        except (ValueError, TypeError):
+                            # 如果不是标量，保持原值
+                            pass
+                    elif hasattr(value, 'value') and not isinstance(value, str):
+                        # pandas 或其他有 value 属性的对象
+                        try:
+                            extracted_value = value.value
+                            # 验证提取的值是数值类型
+                            if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').replace('e', '').replace('E', '').isdigit():
+                                value = extracted_value
+                        except (ValueError, TypeError, AttributeError):
+                            # 如果提取失败，保持原值
+                            pass
                     return float(value)
             except (ValueError, TypeError, InvalidOperation) as e:
                 logger.error(f"值 `{value}` 无法转换为数值类型: {e}", {"库": db_name, "表": table_name, "列": col_name})
@@ -816,8 +881,11 @@ class MySQLUploader:
 
         # 检查是否是百分比字符串
         if isinstance(value, str):
-            if value.endswith('%'):
-                return 'DECIMAL(10,4)'  # 百分比统一使用DECIMAL(10,4)
+            if '%' in value:
+                if re.match(r'^-?\d+(\.\d+)?%$', value.strip()):
+                    return 'DECIMAL(10, 4)'  # 百分比转为小数，使用DECIMAL
+                else:
+                    return 'VARCHAR(255)' # 不符合格式的百分比，视为字符串
 
         if isinstance(value, bool):
             return 'TINYINT(1)'
@@ -861,8 +929,8 @@ class MySQLUploader:
                 return 'MEDIUMTEXT'
             else:
                 return 'LONGTEXT'
-        else:
-            return 'VARCHAR(255)'
+            
+        return 'VARCHAR(255)' # 默认字符串类型
 
     def normalize_column_names(self, data: Union[pd.DataFrame, List[Dict[str, Any]]]) -> Union[
         pd.DataFrame, List[Dict[str, Any]]]:
@@ -1317,7 +1385,7 @@ class MySQLUploader:
         for col in dup_cols:
             col_type = set_typ.get(col, '').lower()
             if col_type.startswith('decimal'):
-                scale = self._get_decimal_scale(col_type)
+                _, scale = self._get_decimal_scale(col_type)
                 conditions.append(f"ROUND(`{col}`, {scale}) = ROUND(%s, {scale})")
             else:
                 conditions.append(f"`{col}` = %s")
@@ -1346,24 +1414,12 @@ class MySQLUploader:
             """
         return sql
 
-    def _get_decimal_scale(self, decimal_type: str) -> int:
-        """
-        从DECIMAL类型定义中提取小数位数
-
-        :param decimal_type: DECIMAL类型字符串，如'DECIMAL(10,4)'
-        :return: 小数位数
-        :raises: 无显式抛出异常，但解析失败时返回默认值2
-        """
-        try:
-            # 匹配DECIMAL类型中的精度和小数位数
-            match = re.match(r'decimal\((\d+),\s*(\d+)\)', decimal_type.lower())
-            if match:
-                return int(match.group(2))
-        except (ValueError, AttributeError, IndexError):
-            pass
-
-        # 默认返回2位小数
-        return 2
+    def _get_decimal_scale(self, decimal_type: str) -> Tuple[int, int]:
+        """从DECIMAL类型字符串中提取精度和标度"""
+        match = re.search(r'\((\d+)\s*,\s*(\d+)\)', decimal_type)
+        if match:
+            return int(match.group(1)), int(match.group(2))
+        return 18, 2  # 默认值
 
     def _prepare_insert_sql(
             self,

{mdbq-4.0.40.dist-info → mdbq-4.0.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.40
+Version: 4.0.42
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.40.dist-info → mdbq-4.0.42.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=YnJ4yS3LCjrBMdYJ8VXs6BM72GyNYWV1Zwk0iW2hB1k,18
+mdbq/__version__.py,sha256=oNzbjnM_gPKVuYbTskX3n8fcwlgDsOEOgDiMJaHcsIU,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=WtTFMN78jn43Y-nBTPAXhAK56w3wDuv_cj4YtzzGbZk,169797
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -11,7 +11,7 @@ mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=1wJ3HVjHEF6FA-bVeeesRlsf73CZSvVTEQ51CF1OsE4,46786
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=FOroXUIsxJaMCqBeepUuymCpdhJsid4yiC_Rs1BT1sw,82823
+mdbq/mysql/uploader.py,sha256=d2ocnCTyBkAJ4LjFLOatb0VgkJ28g70fPXJropRlH1s,86370
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=XptHjGzbout9IYzWAOQUpMMV5qEgLTU8pL1ZGt8oNEA,21868
-mdbq-4.0.40.dist-info/METADATA,sha256=FUd8oXQ4zlMFv9neapfYa24uQl84PHgt3S0aAEbEjGo,364
-mdbq-4.0.40.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.40.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.40.dist-info/RECORD,,
+mdbq-4.0.42.dist-info/METADATA,sha256=XEoFYTnRfRqAv-rIatsbqCFRTXAP9vP0Zv6g-KzIOWU,364
+mdbq-4.0.42.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.42.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.42.dist-info/RECORD,,