mdbq 4.1.10__py3-none-any.whl → 4.1.11__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.1.10'
+VERSION = '4.1.11'

mdbq/mysql/uploader.py

@@ -111,6 +111,7 @@ class MySQLUploader:
         :param read_timeout: 读取超时(秒)，默认为30
         :param write_timeout: 写入超时(秒)，默认为30
         :param ssl: SSL配置字典，默认为None
+        :param auto_creat_missing_cols: 自动添加缺失列，默认为False，建议手动维护表结构
         """
         self.username = username
         self.password = password
@@ -134,6 +135,7 @@ class MySQLUploader:
         self._table_metadata_cache = {}
         self.metadata_cache_ttl = 300  # 5分钟缓存时间
         self.pool = self._create_connection_pool()  # 创建连接池
+        self.auto_creat_missing_cols = False  # 自动添加缺失列，正常不要自动添加，建议手动维护表结构
 
     def _create_connection_pool(self) -> PooledDB:
         """
@@ -695,15 +697,16 @@ class MySQLUploader:
                 break
         
         if not allow_null:
-            logger.warning("该列不允许为空值", {
-                "库": db_name, 
-                "表": table_name, 
-                "allow_null": allow_null, 
-                "列": col_name, 
-                "值": original_value, 
-                "兜底值": fallback
-            })
-            return fallback  # 直接返回兜底值，而不是抛出异常
+            # 注释掉，这里可能会产生大量日志
+            # logger.debug("该列不允许为空值", {
+            #     "库": db_name, 
+            #     "表": table_name, 
+            #     "allow_null": allow_null, 
+            #     "列": col_name, 
+            #     "值": original_value, 
+            #     "兜底值": fallback
+            # })
+            return fallback  # 直接返回兜底值
         
         return None  # 允许空值时返回None
 
@@ -799,7 +802,13 @@ class MySQLUploader:
         
         # 统一处理空值
         if is_empty_value:
-            return self._get_fallback_value(column_type_lower, allow_null, db_name, table_name, col_name, value)
+            fallback_value = self._get_fallback_value(column_type_lower, allow_null, db_name, table_name, col_name, value)
+            # 如果返回了兜底值（非None），直接返回，不再进行后续验证
+            # 因为兜底值已经是根据列类型设计的合适值
+            if fallback_value is not None:
+                return fallback_value
+            # 如果返回None（允许空值的情况），继续后续处理
+            return None
         
         # JSON类型验证和转换
         if 'json' in column_type_lower:
@@ -1072,14 +1081,41 @@ class MySQLUploader:
                 '列': self._shorten_for_log(table_columns),
             })
             raise ValueError(f"获取列失败 `{db_name}`.`{table_name}`")
-        for col in set_typ:
-            if col not in table_columns:
-                logger.error('列不存在', {
+        # 检查并自动添加缺失的列
+        missing_columns = [col for col in set_typ if col not in table_columns]
+        if missing_columns:
+            if not self.auto_creat_missing_cols:
+                logger.error('列不存在且不支持自动添加，请手动维护表结构，并补齐缺失列', {
                     '库': db_name,
                     '表': table_name,
-                    '列': col,
+                    '缺失列数': len(missing_columns),
+                    '缺失列': missing_columns,
                 })
-                raise ValueError(f"列不存在: `{col}` -> `{db_name}`.`{table_name}`")
+                raise ValueError(f"列不存在: `{missing_columns}` -> `{db_name}`.`{table_name}`")
+            else:
+                # 表有缺失列时报错，建议不允许自动添加，手动检查数据一致性，以免产生不必要的表错误
+                # 自动添加缺失的列
+                for col in missing_columns:
+                    try:
+                        self._add_column_to_table(db_name, table_name, col, set_typ[col], allow_null)
+                        logger.info('自动添加缺失列', {
+                            '库': db_name,
+                            '表': table_name,
+                            '列': col,
+                            '类型': set_typ[col]
+                        })
+                    except Exception as e:
+                        logger.error('添加列失败', {
+                            '库': db_name,
+                            '表': table_name,
+                            '列': col,
+                            '类型': set_typ[col],
+                            '错误': str(e)
+                        })
+                        raise ValueError(f"添加列失败: `{col}` -> `{db_name}`.`{table_name}`: {str(e)}")
+                
+                # 重新获取表列信息
+                table_columns = self._get_table_columns(db_name, table_name)
         if date_column and date_column in table_columns:
             try:
                 self._ensure_index(db_name, table_name, date_column)
@@ -1300,21 +1336,67 @@ class MySQLUploader:
                         prepared_row[col_name] = self._validate_value(None, filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
                     except ValueError as e:
                         if not allow_null:
-                            error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
-                            logger.error(error_msg, {'row': self._shorten_for_log(row)})
-                            raise ValueError(error_msg)
-                        prepared_row[col_name] = None
+                            # 如果不允许空值但验证失败，尝试使用兜底值
+                            try:
+                                fallback_value = self._get_fallback_value(filtered_set_typ[col_name].lower(), allow_null, db_name, table_name, col_name, None)
+                                if fallback_value is not None:
+                                    prepared_row[col_name] = fallback_value
+                                    logger.warning(f"行号:{row_idx} -> 缺失列: `{col_name}`, 使用兜底值: {fallback_value}", {'row': self._shorten_for_log(row)})
+                                else:
+                                    error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
+                                    logger.error(error_msg, {'row': self._shorten_for_log(row)})
+                                    raise ValueError(error_msg)
+                            except Exception:
+                                error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
+                                logger.error(error_msg, {'row': self._shorten_for_log(row)})
+                                raise ValueError(error_msg)
+                        else:
+                            prepared_row[col_name] = None
                 else:
                     try:
                         prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
                     except ValueError as e:
-                        logger.error('数据验证失败', {
-                            '列': col_name,
-                            '行': row_idx,
-                            '报错': str(e),
-                            'row': self._shorten_for_log(row),
-                        })
-                        raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
+                        # 如果数据验证失败，检查是否为空值且不允许空值，尝试使用兜底值
+                        original_value = row[col_name]
+                        is_empty_original = (original_value is None or 
+                                           original_value == '' or 
+                                           (not isinstance(original_value, (list, dict)) and 
+                                            pd.isna(original_value) if hasattr(pd, 'isna') else False))
+                        
+                        if is_empty_original and not allow_null:
+                            try:
+                                fallback_value = self._get_fallback_value(filtered_set_typ[col_name].lower(), allow_null, db_name, table_name, col_name, original_value)
+                                if fallback_value is not None:
+                                    prepared_row[col_name] = fallback_value
+                                    logger.warning(f"行:{row_idx}, 列:`{col_name}` -> 原值验证失败，使用兜底值: {fallback_value}", {
+                                        '原值': original_value,
+                                        '兜底值': fallback_value,
+                                        'row': self._shorten_for_log(row)
+                                    })
+                                else:
+                                    logger.error('数据验证失败', {
+                                        '列': col_name,
+                                        '行': row_idx,
+                                        '报错': str(e),
+                                        'row': self._shorten_for_log(row),
+                                    })
+                                    raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
+                            except Exception:
+                                logger.error('数据验证失败', {
+                                    '列': col_name,
+                                    '行': row_idx,
+                                    '报错': str(e),
+                                    'row': self._shorten_for_log(row),
+                                })
+                                raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
+                        else:
+                            logger.error('数据验证失败', {
+                                '列': col_name,
+                                '行': row_idx,
+                                '报错': str(e),
+                                'row': self._shorten_for_log(row),
+                            })
+                            raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
             prepared_data.append(prepared_row)
         return prepared_data, filtered_set_typ
 
@@ -2080,6 +2162,70 @@ class MySQLUploader:
         except Exception as e:
             logger.error('创建索引失败', {'库': db_name, '表': table_name, '列': column, '错误': str(e)})
             raise
+
+    @_execute_with_retry
+    def _add_column_to_table(self, db_name: str, table_name: str, column: str, column_type: str, allow_null: bool = False):
+        """
+        添加列到指定表。
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param column: 需要添加的列名
+        :param column_type: 列的数据类型
+        :param allow_null: 是否允许空值，默认为False
+        """
+        db_name = self._validate_identifier(db_name, is_database=True)
+        table_name = self._validate_identifier(table_name)
+        column = self._validate_identifier(column)
+        
+        # 构建ALTER TABLE语句
+        null_constraint = "NULL" if allow_null else "NOT NULL"
+        
+        # 为新添加的列设置默认值
+        default_value = ""
+        if not allow_null:
+            column_type_lower = column_type.lower()
+            if any(t in column_type_lower for t in ['int', 'bigint', 'tinyint', 'smallint', 'mediumint']):
+                default_value = " DEFAULT 0"
+            elif any(t in column_type_lower for t in ['decimal', 'float', 'double']):
+                default_value = " DEFAULT 0.0"
+            elif any(t in column_type_lower for t in ['varchar', 'text', 'char', 'mediumtext', 'longtext']):
+                default_value = " DEFAULT 'none'"
+            elif 'date' in column_type_lower:
+                if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
+                    default_value = " DEFAULT '1970-01-01 00:00:00'"
+                else:
+                    default_value = " DEFAULT '1970-01-01'"
+            elif 'json' in column_type_lower:
+                default_value = " DEFAULT '{}'"
+        
+        sql = f'ALTER TABLE `{db_name}`.`{table_name}` ADD COLUMN `{column}` {column_type} {null_constraint}{default_value}'
+        
+        conn = None
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                conn.commit()
+                logger.debug('已为表添加列', {
+                    '库': db_name, 
+                    '表': table_name, 
+                    '列': column, 
+                    '类型': column_type,
+                    '允许空值': allow_null
+                })
+        except Exception as e:
+            logger.error('添加列失败', {
+                '库': db_name, 
+                '表': table_name, 
+                '列': column, 
+                '类型': column_type,
+                '错误': str(e),
+                'SQL': sql
+            })
+            if conn is not None:
+                conn.rollback()
+            raise
     
     def __enter__(self):
         return self

{mdbq-4.1.10.dist-info → mdbq-4.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: mdbq
-Version: 4.1.10
+Version: 4.1.11
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.1.10.dist-info → mdbq-4.1.11.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=8l78bszSlbzxxvQ7CZ2KEkqlzv3LZx4D6_I2KFjhHPc,18
+mdbq/__version__.py,sha256=u5gDacgvRqvefEX8ZuCoclKHVFPzAOlN3K-QB3BRCjk,18
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
 mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -15,7 +15,7 @@ mdbq/mysql/deduplicator.py,sha256=2fugLyKs_xkvYvoG0C0hRYbJ_w8-4oa1FJ_vavoD7Qo,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=N2xHJf2CiUXjXIVBemdst-wamIP3908EGAJOFG13fCU,50475
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=xsz2pMPGC2ibT7HMrP3nt3AZ0dRNBisWokclBNls8P4,117716
+mdbq/mysql/uploader.py,sha256=ijeDZ_Z9krLjVltWzcM8JsHev9eBqB9CNcac_W2mWCg,125975
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.1.10.dist-info/METADATA,sha256=2JfWPClIa_RH3GjJVMf69DkPghfLrpVULm6UmbopAlk,364
-mdbq-4.1.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.1.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.1.10.dist-info/RECORD,,
+mdbq-4.1.11.dist-info/METADATA,sha256=uDJZ-MeTKGJLduZ9Pyf-tN5kFgE9ZqYjakSJtrjzt_I,364
+mdbq-4.1.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mdbq-4.1.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.1.11.dist-info/RECORD,,

{mdbq-4.1.10.dist-info → mdbq-4.1.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any