mdbq 4.0.46__py3-none-any.whl → 4.0.47__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.46'
+VERSION = '4.0.47'

mdbq/mysql/uploader.py

@@ -14,6 +14,7 @@ from dbutils.pooled_db import PooledDB
 import sys
 from decimal import Decimal, InvalidOperation
 import math
+import json
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
@@ -630,7 +631,6 @@ class MySQLUploader:
         if 'json' in column_type_lower:
             if isinstance(value, (dict, list)):
                 try:
-                    import json
                     return json.dumps(value, ensure_ascii=False)
                 except (TypeError, ValueError) as e:
                     logger.error(f"JSON序列化失败: {e}", {"库": db_name, "表": table_name, "列": col_name, "值": value})
@@ -638,7 +638,6 @@ class MySQLUploader:
             elif isinstance(value, str):
                 # 验证字符串是否为有效的JSON
                 try:
-                    import json
                     json.loads(value)
                     return value
                 except (TypeError, ValueError) as e:
@@ -743,7 +742,15 @@ class MySQLUploader:
             except (AttributeError, IndexError):
                 pass
             return str_value
-        return value
+        
+        # 兜底处理：确保所有返回值都是基本数据类型
+        if isinstance(value, (dict, list)):
+            try:
+                return json.dumps(value, ensure_ascii=False)
+            except (TypeError, ValueError):
+                return str(value)
+        else:
+            return str(value)
 
     @_execute_with_retry
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
@@ -1551,6 +1558,16 @@ class MySQLUploader:
                 return 1000
             else:
                 return 2000
+        
+        def ensure_basic_type(value):
+            """确保值是基本数据类型，如果是字典或列表则转换为字符串"""
+            if isinstance(value, (dict, list)):
+                try:
+                    return json.dumps(value, ensure_ascii=False)
+                except (TypeError, ValueError):
+                    return str(value)
+            return value
+        
         batch_size = get_optimal_batch_size(len(data))
         all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
         total_inserted = 0
@@ -1563,10 +1580,10 @@ class MySQLUploader:
                         batch = data[i:i + batch_size]
                         values_list = []
                         for row in batch:
-                            values = [row.get(col) for col in all_columns]
+                            values = [ensure_basic_type(row.get(col)) for col in all_columns]
                             if check_duplicate and not update_on_duplicate:
                                 dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
-                                values += [row.get(col) for col in dup_cols]
+                                values += [ensure_basic_type(row.get(col)) for col in dup_cols]
                             values_list.append(values)
                         try:
                             cursor.executemany(sql, values_list)
@@ -1599,10 +1616,10 @@ class MySQLUploader:
                         batch = data[i:i + hybrid_n]
                         for row in batch:
                             try:
-                                values = [row.get(col) for col in all_columns]
+                                values = [ensure_basic_type(row.get(col)) for col in all_columns]
                                 if check_duplicate and not update_on_duplicate:
                                     dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
-                                    values += [row.get(col) for col in dup_cols]
+                                    values += [ensure_basic_type(row.get(col)) for col in dup_cols]
                                 cursor.execute(sql, values)
                                 affected = cursor.rowcount if cursor.rowcount is not None else 0
                                 if update_on_duplicate:
@@ -1627,10 +1644,10 @@ class MySQLUploader:
                 else:  # row模式
                     for row in data:
                         try:
-                            values = [row.get(col) for col in all_columns]
+                            values = [ensure_basic_type(row.get(col)) for col in all_columns]
                             if check_duplicate and not update_on_duplicate:
                                 dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
-                                values += [row.get(col) for col in dup_cols]
+                                values += [ensure_basic_type(row.get(col)) for col in dup_cols]
                             cursor.execute(sql, values)
                             affected = cursor.rowcount if cursor.rowcount is not None else 0
                             if update_on_duplicate:

{mdbq-4.0.46.dist-info → mdbq-4.0.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.46
+Version: 4.0.47
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.46.dist-info → mdbq-4.0.47.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=_sPwKl25zozCOVLAfNKDLrJOiNvhHbBB2Oh22FfZsGk,18
+mdbq/__version__.py,sha256=VOS7S7alTl8woUwMDIW96XJ-1hJ6jsjObiRC2Q4yLis,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=gQUcdrK0QCA0nEBkRA9zBieRWqSnkvpV5Eb-hXKw9Y8,170094
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -11,7 +11,7 @@ mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=1wJ3HVjHEF6FA-bVeeesRlsf73CZSvVTEQ51CF1OsE4,46786
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=NKwuBZcm4597_3gnY9M5m-jnD5joaVjf28twfQfP_jE,87955
+mdbq/mysql/uploader.py,sha256=Y5gCXuhZR-Oo89xaU4wRlcrzDtarABEyJLt43GvDhcI,88718
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=XptHjGzbout9IYzWAOQUpMMV5qEgLTU8pL1ZGt8oNEA,21868
-mdbq-4.0.46.dist-info/METADATA,sha256=gwdYseVCsqG4wlrM-Upti194tEpmc1PB7TKshTGKI30,364
-mdbq-4.0.46.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.46.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.46.dist-info/RECORD,,
+mdbq-4.0.47.dist-info/METADATA,sha256=QhfkX2DquhukBOdBvFoJMZfRoZ6Hcja6VWQxaZGcPz0,364
+mdbq-4.0.47.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.47.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.47.dist-info/RECORD,,