mdbq 4.1.13__py3-none-any.whl → 4.1.14__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.1.13'
+VERSION = '4.1.14'

mdbq/mysql/uploader.py

@@ -1362,79 +1362,60 @@ class MySQLUploader:
                 # 跳过id列，不允许外部传入id
                 if (self.case_sensitive and col_name == 'id') or (not self.case_sensitive and col_name.lower() == 'id'):
                     continue
-                # 对于自动时间戳字段，使用特殊标记让MySQL使用DEFAULT值
+                # 对于自动时间戳字段，跳过处理，让MySQL自动处理
                 col_type_lower = filtered_set_typ[col_name].lower()
                 is_auto_timestamp = ('timestamp' in col_type_lower and 'current_timestamp' in col_type_lower and 
                                    col_name in ['创建时间', '更新时间'])
                 
+                if is_auto_timestamp:
+                    # 自动时间戳字段完全跳过，不在INSERT语句中包含
+                    continue
+                
                 if col_name not in row:
                     # 对于缺失的列，使用None作为默认值，在_validate_value中会根据allow_null和列类型进行进一步处理
-                    if is_auto_timestamp:
-                        # 自动时间戳字段使用特殊标记
-                        prepared_row[col_name] = 'DEFAULT'
-                    else:
-                        try:
-                            prepared_row[col_name] = self._validate_value(None, filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
-                        except ValueError as e:
-                            if not allow_null:
-                                # 如果不允许空值但验证失败，尝试使用兜底值
-                                try:
-                                    fallback_value = self._get_fallback_value(filtered_set_typ[col_name].lower(), allow_null, db_name, table_name, col_name, None)
-                                    if fallback_value is not None:
-                                        prepared_row[col_name] = fallback_value
-                                        logger.warning(f"行号:{row_idx} -> 缺失列: `{col_name}`, 使用兜底值: {fallback_value}", {'row': self._shorten_for_log(row)})
-                                    else:
-                                        error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
-                                        logger.error(error_msg, {'row': self._shorten_for_log(row)})
-                                        raise ValueError(error_msg)
-                                except Exception:
+                    try:
+                        prepared_row[col_name] = self._validate_value(None, filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
+                    except ValueError as e:
+                        if not allow_null:
+                            # 如果不允许空值但验证失败，尝试使用兜底值
+                            try:
+                                fallback_value = self._get_fallback_value(filtered_set_typ[col_name].lower(), allow_null, db_name, table_name, col_name, None)
+                                if fallback_value is not None:
+                                    prepared_row[col_name] = fallback_value
+                                    logger.warning(f"行号:{row_idx} -> 缺失列: `{col_name}`, 使用兜底值: {fallback_value}", {'row': self._shorten_for_log(row)})
+                                else:
                                     error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
                                     logger.error(error_msg, {'row': self._shorten_for_log(row)})
                                     raise ValueError(error_msg)
-                            else:
-                                prepared_row[col_name] = None
+                            except Exception:
+                                error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
+                                logger.error(error_msg, {'row': self._shorten_for_log(row)})
+                                raise ValueError(error_msg)
+                        else:
+                            prepared_row[col_name] = None
                 else:
-                    if is_auto_timestamp:
-                        # 自动时间戳字段忽略用户传入的值，使用DEFAULT
-                        prepared_row[col_name] = 'DEFAULT'
-                        if row[col_name] is not None:  # 如果用户传入了值，给出警告
-                            logger.warning('忽略自动时间戳字段的用户传入值', {
-                                '库': db_name,
-                                '表': table_name,
-                                '列': col_name,
-                                '用户值': row[col_name],
-                                '原因': '将使用MySQL CURRENT_TIMESTAMP'
-                            })
-                    else:
-                        try:
-                            prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
-                        except ValueError as e:
-                            # 如果数据验证失败，检查是否为空值且不允许空值，尝试使用兜底值
-                            original_value = row[col_name]
-                            is_empty_original = (original_value is None or 
-                                               original_value == '' or 
-                                               (not isinstance(original_value, (list, dict)) and 
-                                                pd.isna(original_value) if hasattr(pd, 'isna') else False))
-                            
-                            if is_empty_original and not allow_null:
-                                try:
-                                    fallback_value = self._get_fallback_value(filtered_set_typ[col_name].lower(), allow_null, db_name, table_name, col_name, original_value)
-                                    if fallback_value is not None:
-                                        prepared_row[col_name] = fallback_value
-                                        logger.warning(f"行:{row_idx}, 列:`{col_name}` -> 原值验证失败，使用兜底值: {fallback_value}", {
-                                            '原值': original_value,
-                                            '兜底值': fallback_value,
-                                            'row': self._shorten_for_log(row)
-                                        })
-                                    else:
-                                        logger.error('数据验证失败', {
-                                            '列': col_name,
-                                            '行': row_idx,
-                                            '报错': str(e),
-                                            'row': self._shorten_for_log(row),
-                                        })
-                                        raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
-                                except Exception:
+                    # 处理用户传入的值
+                    try:
+                        prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
+                    except ValueError as e:
+                        # 如果数据验证失败，检查是否为空值且不允许空值，尝试使用兜底值
+                        original_value = row[col_name]
+                        is_empty_original = (original_value is None or 
+                                           original_value == '' or 
+                                           (not isinstance(original_value, (list, dict)) and 
+                                            pd.isna(original_value) if hasattr(pd, 'isna') else False))
+                        
+                        if is_empty_original and not allow_null:
+                            try:
+                                fallback_value = self._get_fallback_value(filtered_set_typ[col_name].lower(), allow_null, db_name, table_name, col_name, original_value)
+                                if fallback_value is not None:
+                                    prepared_row[col_name] = fallback_value
+                                    logger.warning(f"行:{row_idx}, 列:`{col_name}` -> 原值验证失败，使用兜底值: {fallback_value}", {
+                                        '原值': original_value,
+                                        '兜底值': fallback_value,
+                                        'row': self._shorten_for_log(row)
+                                    })
+                                else:
                                     logger.error('数据验证失败', {
                                         '列': col_name,
                                         '行': row_idx,
@@ -1442,7 +1423,7 @@ class MySQLUploader:
                                         'row': self._shorten_for_log(row),
                                     })
                                     raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
-                            else:
+                            except Exception:
                                 logger.error('数据验证失败', {
                                     '列': col_name,
                                     '行': row_idx,
@@ -1450,6 +1431,14 @@ class MySQLUploader:
                                     'row': self._shorten_for_log(row),
                                 })
                                 raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
+                        else:
+                            logger.error('数据验证失败', {
+                                '列': col_name,
+                                '行': row_idx,
+                                '报错': str(e),
+                                'row': self._shorten_for_log(row),
+                            })
+                            raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
                 prepared_data.append(prepared_row)
         return prepared_data, filtered_set_typ
 
@@ -1899,8 +1888,17 @@ class MySQLUploader:
         cached = self._prepared_statements.get(cache_key)
         if cached:
             return cached
-        # 获取所有列名（排除id）
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+        # 获取所有列名（排除id和自动时间戳字段）
+        all_columns = []
+        for col in set_typ.keys():
+            if col.lower() == 'id':
+                continue
+            # 检查是否是自动时间戳字段
+            col_type_lower = set_typ[col].lower()
+            is_auto_timestamp = ('timestamp' in col_type_lower and 'current_timestamp' in col_type_lower and 
+                               col in ['创建时间', '更新时间'])
+            if not is_auto_timestamp:
+                all_columns.append(col)
         if not check_duplicate:
             sql = self._build_simple_insert_sql(db_name, table_name, all_columns,
                                                  update_on_duplicate)
@@ -1956,51 +1954,19 @@ class MySQLUploader:
                     return str(value)
             return value
         
-        def execute_single_row_with_defaults(row):
-            """处理单行插入，支持DEFAULT字段"""
-            has_defaults = any(row.get(col) == 'DEFAULT' for col in all_columns)
-            
-            if has_defaults:
-                # 分离普通字段和DEFAULT字段
-                regular_columns = []
-                regular_values = []
-                default_columns = []
-                
-                for col in all_columns:
-                    val = row.get(col)
-                    if val == 'DEFAULT':
-                        default_columns.append(col)
-                    else:
-                        regular_columns.append(col)
-                        regular_values.append(ensure_basic_type(val))
-                
-                # 构建INSERT ... SET语句
-                set_clauses = []
-                for col in regular_columns:
-                    set_clauses.append(f"`{self._validate_identifier(col)}` = %s")
-                for col in default_columns:
-                    set_clauses.append(f"`{self._validate_identifier(col)}` = DEFAULT")
-                
-                if set_clauses:
-                    dynamic_sql = f"INSERT INTO `{db_name}`.`{table_name}` SET {', '.join(set_clauses)}"
-                    if update_on_duplicate and regular_columns:
-                        update_clauses = [f"`{self._validate_identifier(col)}` = VALUES(`{self._validate_identifier(col)}`)" for col in regular_columns]
-                        if update_clauses:
-                            dynamic_sql += f" ON DUPLICATE KEY UPDATE {', '.join(update_clauses)}"
-                    
-                    cursor.execute(dynamic_sql, regular_values)
-                    return cursor.rowcount if cursor.rowcount is not None else 0
-            else:
-                # 没有DEFAULT字段，使用原有逻辑
-                values = [ensure_basic_type(row.get(col)) for col in all_columns]
-                if check_duplicate and not update_on_duplicate:
-                    dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
-                    values += [ensure_basic_type(row.get(col)) for col in dup_cols]
-                cursor.execute(sql, values)
-                return cursor.rowcount if cursor.rowcount is not None else 0
         
         batch_size = get_optimal_batch_size(len(data))
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+        # 排除id列和自动时间戳列
+        all_columns = []
+        for col in set_typ.keys():
+            if col.lower() == 'id':
+                continue
+            # 检查是否是自动时间戳字段
+            col_type_lower = set_typ[col].lower()
+            is_auto_timestamp = ('timestamp' in col_type_lower and 'current_timestamp' in col_type_lower and 
+                               col in ['创建时间', '更新时间'])
+            if not is_auto_timestamp:
+                all_columns.append(col)
         total_inserted = 0
         total_skipped = 0
         total_failed = 0
@@ -2009,72 +1975,51 @@ class MySQLUploader:
                 if transaction_mode == 'batch':
                     for i in range(0, len(data), batch_size):
                         batch = data[i:i + batch_size]
-                        # 检查是否有DEFAULT字段，如果有则需要特殊处理
-                        has_default_fields = any(row.get(col) == 'DEFAULT' for row in batch for col in all_columns)
-                        
-                        if has_default_fields:
-                            # 对于包含DEFAULT字段的情况，逐行处理
-                            for row in batch:
-                                try:
-                                    affected = execute_single_row_with_defaults(row)
-                                    if update_on_duplicate:
-                                        total_inserted += 1
-                                    else:
-                                        if affected > 0:
-                                            total_inserted += 1
-                                        else:
-                                            total_skipped += 1
-                                except pymysql.err.IntegrityError:
-                                    total_skipped += 1
-                                except Exception as e:
-                                    total_failed += 1
-                                    logger.error('单行插入失败', {
-                                        '库': db_name, 
-                                        '表': table_name, 
-                                        '错误': str(e)
-                                    })
+                        # 使用批量插入逻辑
+                        values_list = []
+                        for row in batch:
+                            values = [ensure_basic_type(row.get(col)) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
+                                values += [ensure_basic_type(row.get(col)) for col in dup_cols]
+                            values_list.append(values)
+                        try:
+                            cursor.executemany(sql, values_list)
                             conn.commit()
-                        else:
-                            # 没有DEFAULT字段，使用原有逻辑
-                            values_list = []
-                            for row in batch:
-                                values = [ensure_basic_type(row.get(col)) for col in all_columns]
-                                if check_duplicate and not update_on_duplicate:
-                                    dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
-                                    values += [ensure_basic_type(row.get(col)) for col in dup_cols]
-                                values_list.append(values)
-                            try:
-                                cursor.executemany(sql, values_list)
-                                conn.commit()
-                                # 在batch模式下，affected_rows表示实际影响的行数
-                                # 如果update_on_duplicate为True，则affected_rows包含更新的行数
-                                # 如果update_on_duplicate为False，则affected_rows只包含插入的行数
-                                affected = cursor.rowcount if cursor.rowcount is not None else 0
-                                if update_on_duplicate:
-                                    # 当启用更新时，affected_rows包含插入和更新的行数
-                                    # 我们需要区分插入和更新的行数
-                                    # 由于无法准确区分，我们假设所有行都是插入的
-                                    total_inserted += len(batch)
-                                else:
-                                    # 当不启用更新时，affected_rows只包含插入的行数
-                                    total_inserted += affected
-                                    total_skipped += len(batch) - affected
-                            except pymysql.err.IntegrityError as e:
-                                conn.rollback()
-                                # 在唯一约束冲突时，所有行都被跳过
-                                total_skipped += len(batch)
-                                logger.debug('批量插入唯一约束冲突，全部跳过', {'库': db_name, '表': table_name, '错误': str(e)})
-                            except Exception as e:
-                                conn.rollback()
-                                total_failed += len(batch)
-                                logger.error('批量插入失败', {'库': db_name, '表': table_name, '错误': str(e)})
+                            # 在batch模式下，affected_rows表示实际影响的行数
+                            # 如果update_on_duplicate为True，则affected_rows包含更新的行数
+                            # 如果update_on_duplicate为False，则affected_rows只包含插入的行数
+                            affected = cursor.rowcount if cursor.rowcount is not None else 0
+                            if update_on_duplicate:
+                                # 当启用更新时，affected_rows包含插入和更新的行数
+                                # 我们需要区分插入和更新的行数
+                                # 由于无法准确区分，我们假设所有行都是插入的
+                                total_inserted += len(batch)
+                            else:
+                                # 当不启用更新时，affected_rows只包含插入的行数
+                                total_inserted += affected
+                                total_skipped += len(batch) - affected
+                        except pymysql.err.IntegrityError as e:
+                            conn.rollback()
+                            # 在唯一约束冲突时，所有行都被跳过
+                            total_skipped += len(batch)
+                            logger.debug('批量插入唯一约束冲突，全部跳过', {'库': db_name, '表': table_name, '错误': str(e)})
+                        except Exception as e:
+                            conn.rollback()
+                            total_failed += len(batch)
+                            logger.error('批量插入失败', {'库': db_name, '表': table_name, '错误': str(e)})
                 elif transaction_mode == 'hybrid':
                     hybrid_n = 100  # 可配置
                     for i in range(0, len(data), hybrid_n):
                         batch = data[i:i + hybrid_n]
                         for row in batch:
                             try:
-                                affected = execute_single_row_with_defaults(row)
+                                values = [ensure_basic_type(row.get(col)) for col in all_columns]
+                                if check_duplicate and not update_on_duplicate:
+                                    dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
+                                    values += [ensure_basic_type(row.get(col)) for col in dup_cols]
+                                cursor.execute(sql, values)
+                                affected = cursor.rowcount if cursor.rowcount is not None else 0
                                 if update_on_duplicate:
                                     # 当启用更新时，affected_rows包含插入和更新的行数
                                     # 假设所有行都是插入的，因为无法区分插入和更新
@@ -2097,7 +2042,12 @@ class MySQLUploader:
                 else:  # row模式
                     for row in data:
                         try:
-                            affected = execute_single_row_with_defaults(row)
+                            values = [ensure_basic_type(row.get(col)) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
+                                values += [ensure_basic_type(row.get(col)) for col in dup_cols]
+                            cursor.execute(sql, values)
+                            affected = cursor.rowcount if cursor.rowcount is not None else 0
                             if update_on_duplicate:
                                 # 当启用更新时，affected_rows包含插入和更新的行数
                                 # 假设所有行都是插入的，因为无法区分插入和更新
@@ -2737,9 +2687,9 @@ class MySQLUploader:
         
         # 使用MySQL的CURRENT_TIMESTAMP功能，按固定顺序添加时间戳列
         # 创建时间：插入时自动设置，更新时不变
-        updated_set_typ[created_col] = 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'
+        updated_set_typ[created_col] = 'TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP'
         # 更新时间：插入和更新时都自动设置为当前时间
-        updated_set_typ[updated_col] = 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'
+        updated_set_typ[updated_col] = 'TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'
         
         # 处理DataFrame格式的数据
         if hasattr(data, 'shape') and hasattr(data, 'columns'):

{mdbq-4.1.13.dist-info → mdbq-4.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdbq
-Version: 4.1.13
+Version: 4.1.14
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.1.13.dist-info → mdbq-4.1.14.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=ekxTFUNsVfy6qrG8u7tQM8OlFtSVSQTGZsMd31HqV9I,18
+mdbq/__version__.py,sha256=zj2uAJqL04jV1wMjxAyT-2T4sjHxXiGCOfNFXJfgjz0,18
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
 mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -15,7 +15,7 @@ mdbq/mysql/deduplicator.py,sha256=tzLIm9K9S0lGLlVTI0dDQVYpWX796XCuyufmw1lU26Y,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=N2xHJf2CiUXjXIVBemdst-wamIP3908EGAJOFG13fCU,50475
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=0ZnC7KxF1y5-UlCPyrdg-MHpVm_NTHdZ6yY2ytxG8Io,136520
+mdbq/mysql/uploader.py,sha256=5zpvcQpa1BAtFKkVF-EQYXSieeF2hHoZQq6lmEaEnjY,133474
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.1.13.dist-info/METADATA,sha256=GJ4JaVRELhrQqlakZeZ_C2swBtLhhVXsd9ens555dck,364
-mdbq-4.1.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mdbq-4.1.13.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.1.13.dist-info/RECORD,,
+mdbq-4.1.14.dist-info/METADATA,sha256=KTtSRhxxSKyWNWlEvHI7YIhD6XDneGbuEw3Qa1ku5EE,364
+mdbq-4.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mdbq-4.1.14.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.1.14.dist-info/RECORD,,