mdbq 4.0.2__py3-none-any.whl → 4.0.4__py3-none-any.whl

mdbq/mysql/uploader.py

@@ -14,6 +14,7 @@ from dbutils.pooled_db import PooledDB
 import json
 import sys
 from decimal import Decimal, InvalidOperation
+import math
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
@@ -240,8 +241,16 @@ class MySQLUploader:
             conn = self.pool.connection()
             return conn
         except Exception as e:
-            logger.error('从连接池获取数据库连接失败', {'error': str(e)})
-            raise ConnectionError(f'连接数据库失败: {str(e)}')
+            logger.error('从连接池获取数据库连接失败，尝试重建连接池', {'error': str(e)})
+            # 强制重建连接池
+            try:
+                self.pool = self._create_connection_pool()
+                conn = self.pool.connection()
+                logger.info('重建连接池后获取连接成功')
+                return conn
+            except Exception as e2:
+                logger.error('重建连接池后依然获取连接失败', {'error': str(e2)})
+                raise ConnectionError(f'连接数据库失败: {str(e2)}')
 
     @_execute_with_retry
     def _check_database_exists(self, db_name: str) -> bool:
@@ -407,31 +416,36 @@ class MySQLUploader:
                 col_def += " NOT NULL"
             column_defs.append(col_def)
         # 主键处理逻辑调整
+        def _index_col_sql(col):
+            col_type = set_typ.get(col, '').lower()
+            if 'varchar' in col_type or 'text' in col_type:
+                return f"`{self._normalize_col(col)}`(100)"
+            return f"`{self._normalize_col(col)}`"
         if primary_keys and len(primary_keys) > 0:
-            safe_primary_keys = [self._normalize_col(pk) for pk in primary_keys]
-            primary_key_sql = f"PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+            safe_primary_keys = [_index_col_sql(pk) for pk in primary_keys]
+            primary_key_sql = f"PRIMARY KEY ({','.join(safe_primary_keys)})"
         else:
-            safe_primary_keys = [self._normalize_col('id')]
+            safe_primary_keys = [_index_col_sql('id')]
             primary_key_sql = f"PRIMARY KEY (`id`)"
         # 索引统一在CREATE TABLE中定义
         index_defs = []
         if date_column and date_column in set_typ:
-            safe_date_col = self._normalize_col(date_column)
-            index_defs.append(f"INDEX `idx_{safe_date_col}` (`{safe_date_col}`)")
+            safe_date_col = _index_col_sql(date_column)
+            index_defs.append(f"INDEX `idx_{self._normalize_col(date_column)}` ({safe_date_col})")
         if indexes:
             for idx_col in indexes:
                 if idx_col in set_typ:
-                    safe_idx_col = self._normalize_col(idx_col)
-                    index_defs.append(f"INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)")
+                    safe_idx_col = _index_col_sql(idx_col)
+                    index_defs.append(f"INDEX `idx_{self._normalize_col(idx_col)}` ({safe_idx_col})")
         # UNIQUE KEY定义
         unique_defs = []
         if unique_keys:
             for unique_cols in unique_keys:
                 if not unique_cols:
                     continue
-                safe_unique_cols = [self._normalize_col(col) for col in unique_cols]
-                unique_name = f"uniq_{'_'.join(safe_unique_cols)}"
-                unique_defs.append(f"UNIQUE KEY `{unique_name}` (`{'`,`'.join(safe_unique_cols)}`)")
+                safe_unique_cols = [_index_col_sql(col) for col in unique_cols]
+                unique_name = f"uniq_{'_'.join([self._normalize_col(c) for c in unique_cols])}"
+                unique_defs.append(f"UNIQUE KEY `{unique_name}` ({','.join(safe_unique_cols)})")
         index_defs = list(set(index_defs))
         all_defs = column_defs + [primary_key_sql] + index_defs + unique_defs
         sql = f"""
@@ -447,7 +461,7 @@ class MySQLUploader:
                 conn.commit()
                 logger.info('数据表及索引已创建', {'库': db_name, '表': table_name, '索引': indexes, '唯一约束': unique_keys})
         except Exception as e:
-            logger.error('建表失败', {'库': db_name, '表': table_name, '错误': str(e)})
+            logger.error('建表失败', {'库': db_name, '表': table_name, '错误': str(e), '异常类型': type(e).__name__})
             if conn is not None:
                 conn.rollback()
             raise
@@ -491,25 +505,45 @@ class MySQLUploader:
     def _validate_value(self, value: Any, column_type: str, allow_null: bool, db_name: str = None, table_name: str = None, col_name: str = None) -> Any:
         """
         根据列类型验证并转换数据值
-
-        :param value: 要验证的值
-        :param column_type: 列的数据类型
-        :param allow_null: 是否允许空值
-        :param db_name: 数据库名（用于日志）
-        :param table_name: 表名（用于日志）
-        :param col_name: 列名（用于日志）
-        :return: 转换后的值
-        :raises ValueError: 当值转换失败时抛出
         """
+        column_type_lower = column_type.lower() if column_type else ''
+        # 统一判断None/NaN
+        is_nan = False
         if value is None:
+            is_nan = True
+        elif isinstance(value, float) and math.isnan(value):
+            is_nan = True
+        elif str(value).lower() in ['nan', 'none']:
+            is_nan = True
+        if is_nan:
             if not allow_null:
-                logger.warning('字段值为None但不允许空值, 已填充为none', {
-                    '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
-                })
-                return 'none'
+                if 'int' in column_type_lower:
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                    })
+                    return 0
+                elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为0.0', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                    })
+                    return 0.0
+                elif 'date' in column_type_lower or 'time' in column_type_lower:
+                    # 判断是date还是datetime/timestamp
+                    if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
+                        default_date = '2000-01-01 00:00:00'
+                    else:
+                        default_date = '2000-01-01'
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为默认日期', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type, '默认值': default_date
+                    })
+                    return default_date
+                else:
+                    logger.debug('字段值为None/NaN但不允许空值, 已填充为none字符串', {
+                        '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                    })
+                    return 'none'
             return None
         try:
-            column_type_lower = column_type.lower()
             if isinstance(value, str) and value.strip().endswith('%'):
                 try:
                     percent_str = value.strip().replace('%', '')
@@ -881,22 +915,21 @@ class MySQLUploader:
         # set_typ的键清洗
         set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
 
-        # 获取数据中实际存在的列名
-        data_columns = set()
-        if data and len(data) > 0:
-            data_columns = set(data[0].keys())
-
-        # 过滤set_typ，只保留数据中存在的列
+        # 新实现：严格按set_typ顺序过滤，后补充data中有但set_typ没有的列
         filtered_set_typ = {}
-        for col in data_columns:
-            if col in set_typ:
+        data_columns = list(data[0].keys()) if data and len(data) > 0 else []
+        # 先按set_typ顺序
+        for col in set_typ:
+            if col in data_columns:
                 filtered_set_typ[col] = set_typ[col]
-            else:
-                # 如果列不在set_typ中，采样多个非None值推断类型
+        # 再补充data中有但set_typ没有的列
+        for col in data_columns:
+            if col not in filtered_set_typ:
+                # 推断类型
                 sample_values = [row[col] for row in data if col in row and row[col] is not None][:5]
                 inferred_type = None
                 for val in sample_values:
-                    inferred_type = self._infer_data_type(val, no_log=True)  # 推断日期类型不记录日志, 避免日志噪音过多
+                    inferred_type = self._infer_data_type(val, no_log=True)
                     if inferred_type:
                         break
                 if not inferred_type:
@@ -1326,8 +1359,7 @@ class MySQLUploader:
         if cached:
             return cached
         # 获取所有列名（排除id）
-        all_columns = [col for col in set_typ.keys()
-                       if col.lower() != 'id']
+        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
         if not check_duplicate:
             sql = self._build_simple_insert_sql(db_name, table_name, all_columns,
                                                  update_on_duplicate)
@@ -1364,7 +1396,6 @@ class MySQLUploader:
         - 只有遇到严重的数据库错误（如所有行都因唯一约束冲突且没有ON DUPLICATE KEY UPDATE），才会整体回滚。
         - 返回值为(插入行数, 跳过行数, 失败行数)。
         """
-        import pymysql  # 确保异常类型可用
         def get_optimal_batch_size(total_rows: int) -> int:
             if total_rows <= 100:
                 return total_rows
@@ -1612,5 +1643,5 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    # main()
     pass

{mdbq-4.0.2.dist-info → mdbq-4.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.2
+Version: 4.0.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.2.dist-info → mdbq-4.0.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=_fU0Mj16CzZHiwvkoGULFRC0vc6b0FxDy6MgQQON3Gw,17
+mdbq/__version__.py,sha256=lAYjWBa6ThlPWc3_1b0lWM6fn_3Z9ckuALnMMZXhBbs,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=U6dYK8_gEaNnsGKooEkzfAWnzNA8kt0uomec49e4olE,177536
+mdbq/aggregation/query_data.py,sha256=_k6Jg60RaaT056sIaiSO6v84dEnOIOGq-nUJtSr65kI,171861
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -10,9 +10,9 @@ mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,16
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=8v3MC6TJ0YEiExWrTP9OXAxTYnL9XbpYL2vWaER1h2M,73099
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=tSBEbyuVQBeE6tckHSbguAQh9T07tvPhf4J6DlpUBP8,10508
+mdbq/mysql/s_query.py,sha256=RnVCwMQ_n9PcAimbMWbHe9k8eil8shtCfa3LwLBZi6c,41909
 mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=n6K2rn7cGHGLGevX2JO7pKrUiQiGNluRx3CwnxARZGI,72503
+mdbq/mysql/uploader.py,sha256=e49Gk09K766QXaus_p3VOMcH2VbexQzKsqDTCGrWoWQ,74419
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=l3zBK7wrZl0oO42-_UGylyatnIp_SBw8wDDvof9fht4,23534
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=hPRzLQvFIF4ibN8aP3Dg_ru5meac90faPyzOB22cj-o,20965
-mdbq-4.0.2.dist-info/METADATA,sha256=EdCDmGum97Hjm9aKPV-2qAf9IoMkh-F0DE6ATMLJTX8,363
-mdbq-4.0.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.2.dist-info/RECORD,,
+mdbq-4.0.4.dist-info/METADATA,sha256=VM2dtOiBJ74NlYhq9UWAFAPloayAXPX9bLKnvZJd7Xg,363
+mdbq-4.0.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.4.dist-info/RECORD,,