mdbq 4.1.12__py3-none-any.whl → 4.1.14__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.1.12'
+VERSION = '4.1.14'

mdbq/mysql/deduplicator.py

@@ -1364,12 +1364,12 @@ def main():
         skip_system_dbs=True,
         max_retries=3,
         retry_waiting_time=5,
-        pool_size=20,
-        mincached=5,
-        maxcached=10,
+        pool_size=10,
+        mincached=2,
+        maxcached=5,
         # recent_month=1,
         # date_range=['2025-06-09', '2025-06-10'],
-        exclude_columns=['更新时间'],
+        exclude_columns=['创建时间', '更新时间'],
         exclude_databases=['cookie文件', '日志', '视频数据', '云电影'],
         # exclude_tables={
         #     '推广数据2': [
@@ -1391,9 +1391,9 @@ def main():
 
     # # 指定表去重(使用特定列)
     deduplicator.deduplicate_table(
-        '生意参谋3', 
-        '新品追踪_旧接口_2025', 
-        columns=['商品id', '累计商品浏览量', '累计商品访客数'], 
+        '推广数据_奥莱店', 
+        '主体报表_2025', 
+        columns=['日期', '店铺名称', '场景id', '计划id', '主体id'], 
         dry_run=False, 
         reorder_id=True,
         )

mdbq/mysql/uploader.py

@@ -435,15 +435,36 @@ class MySQLUploader:
         if not primary_keys:
             column_defs.append("`id` INT NOT NULL AUTO_INCREMENT")
         
-        # 添加其他列
+        # 添加其他列，确保时间戳字段按正确顺序添加
+        timestamp_cols = ['创建时间', '更新时间']
+        regular_cols = []
+        timestamp_defs = []
+        
+        # 先处理非时间戳字段
         for col_name, col_type in set_typ.items():
             if col_name == 'id':
                 continue
+            if col_name in timestamp_cols:
+                continue  # 时间戳字段稍后按顺序处理
+            
             safe_col_name = self._normalize_col(col_name)
             col_def = f"`{safe_col_name}` {col_type}"
-            if not allow_null and not col_type.lower().startswith('json'):
+            if not allow_null and not col_type.lower().startswith('json') and not col_type.lower().startswith('timestamp'):
                 col_def += " NOT NULL"
-            column_defs.append(col_def)
+            regular_cols.append(col_def)
+        
+        # 按固定顺序添加时间戳字段
+        for timestamp_col in timestamp_cols:
+            if timestamp_col in set_typ:
+                safe_col_name = self._normalize_col(timestamp_col)
+                col_type = set_typ[timestamp_col]
+                col_def = f"`{safe_col_name}` {col_type}"
+                # TIMESTAMP字段不需要额外的NOT NULL，因为已经包含在类型定义中
+                timestamp_defs.append(col_def)
+        
+        # 合并所有列定义：常规字段 + 时间戳字段
+        column_defs.extend(regular_cols)
+        column_defs.extend(timestamp_defs)
             
         # 主键处理逻辑调整
         def _index_col_sql(col):
@@ -672,9 +693,9 @@ class MySQLUploader:
             'decimal': 0.0,
             'float': 0.0,
             'double': 0.0,
-            'date': '1970-01-01',
-            'datetime': '1970-01-01 00:00:00',
-            'timestamp': '1970-01-01 00:00:00',
+            'date': '2000-01-01',
+            'datetime': '2000-01-01 00:00:00',
+            'timestamp': '2000-01-01 00:00:00',
             'json': '{}',
             'varchar': 'none',
             'text': 'none',
@@ -783,6 +804,12 @@ class MySQLUploader:
         """
         column_type_lower = column_type.lower() if column_type else ''
         
+        # 对于包含CURRENT_TIMESTAMP的TIMESTAMP字段，跳过验证，让MySQL自动处理
+        if ('timestamp' in column_type_lower and 'current_timestamp' in column_type_lower and 
+            col_name in ['创建时间', '更新时间']):
+            # 这些字段由MySQL自动处理，不需要传入值
+            return None
+        
         # 统一的空值检查（None、空字符串、NaN）
         is_empty_value = False
         if value is None:
@@ -1335,6 +1362,15 @@ class MySQLUploader:
                 # 跳过id列，不允许外部传入id
                 if (self.case_sensitive and col_name == 'id') or (not self.case_sensitive and col_name.lower() == 'id'):
                     continue
+                # 对于自动时间戳字段，跳过处理，让MySQL自动处理
+                col_type_lower = filtered_set_typ[col_name].lower()
+                is_auto_timestamp = ('timestamp' in col_type_lower and 'current_timestamp' in col_type_lower and 
+                                   col_name in ['创建时间', '更新时间'])
+                
+                if is_auto_timestamp:
+                    # 自动时间戳字段完全跳过，不在INSERT语句中包含
+                    continue
+                
                 if col_name not in row:
                     # 对于缺失的列，使用None作为默认值，在_validate_value中会根据allow_null和列类型进行进一步处理
                     try:
@@ -1358,6 +1394,7 @@ class MySQLUploader:
                         else:
                             prepared_row[col_name] = None
                 else:
+                    # 处理用户传入的值
                     try:
                         prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
                     except ValueError as e:
@@ -1402,7 +1439,7 @@ class MySQLUploader:
                                 'row': self._shorten_for_log(row),
                             })
                             raise ValueError(f"行:{row_idx}, 列:`{col_name}`-> 报错: {str(e)}")
-            prepared_data.append(prepared_row)
+                prepared_data.append(prepared_row)
         return prepared_data, filtered_set_typ
 
     def upload_data(
@@ -1851,8 +1888,17 @@ class MySQLUploader:
         cached = self._prepared_statements.get(cache_key)
         if cached:
             return cached
-        # 获取所有列名（排除id）
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+        # 获取所有列名（排除id和自动时间戳字段）
+        all_columns = []
+        for col in set_typ.keys():
+            if col.lower() == 'id':
+                continue
+            # 检查是否是自动时间戳字段
+            col_type_lower = set_typ[col].lower()
+            is_auto_timestamp = ('timestamp' in col_type_lower and 'current_timestamp' in col_type_lower and 
+                               col in ['创建时间', '更新时间'])
+            if not is_auto_timestamp:
+                all_columns.append(col)
         if not check_duplicate:
             sql = self._build_simple_insert_sql(db_name, table_name, all_columns,
                                                  update_on_duplicate)
@@ -1908,8 +1954,19 @@ class MySQLUploader:
                     return str(value)
             return value
         
+        
         batch_size = get_optimal_batch_size(len(data))
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+        # 排除id列和自动时间戳列
+        all_columns = []
+        for col in set_typ.keys():
+            if col.lower() == 'id':
+                continue
+            # 检查是否是自动时间戳字段
+            col_type_lower = set_typ[col].lower()
+            is_auto_timestamp = ('timestamp' in col_type_lower and 'current_timestamp' in col_type_lower and 
+                               col in ['创建时间', '更新时间'])
+            if not is_auto_timestamp:
+                all_columns.append(col)
         total_inserted = 0
         total_skipped = 0
         total_failed = 0
@@ -1918,6 +1975,7 @@ class MySQLUploader:
                 if transaction_mode == 'batch':
                     for i in range(0, len(data), batch_size):
                         batch = data[i:i + batch_size]
+                        # 使用批量插入逻辑
                         values_list = []
                         for row in batch:
                             values = [ensure_basic_type(row.get(col)) for col in all_columns]
@@ -2209,14 +2267,22 @@ class MySQLUploader:
                 default_value = " DEFAULT 0.0"
             elif any(t in column_type_lower for t in ['varchar', 'text', 'char', 'mediumtext', 'longtext']):
                 default_value = " DEFAULT 'none'"
+            elif 'timestamp' in column_type_lower:
+                # TIMESTAMP类型已经包含DEFAULT定义，不需要额外添加
+                default_value = ""
             elif 'date' in column_type_lower:
-                if 'datetime' in column_type_lower or 'timestamp' in column_type_lower:
-                    default_value = " DEFAULT '1970-01-01 00:00:00'"
+                if 'datetime' in column_type_lower:
+                    default_value = " DEFAULT '2000-01-01 00:00:00'"
                 else:
-                    default_value = " DEFAULT '1970-01-01'"
+                    default_value = " DEFAULT '2000-01-01'"
             elif 'json' in column_type_lower:
                 default_value = " DEFAULT '{}'"
         
+        # 对于TIMESTAMP类型，不添加额外的NULL约束，因为已经包含在类型定义中
+        if 'timestamp' in column_type.lower() and ('default' in column_type.lower() or 'current_timestamp' in column_type.lower()):
+            null_constraint = ""  # TIMESTAMP类型已经包含完整定义
+            default_value = ""
+        
         sql = f'ALTER TABLE `{db_name}`.`{table_name}` ADD COLUMN `{column}` {column_type} {null_constraint}{default_value}'
         
         conn = None
@@ -2615,39 +2681,31 @@ class MySQLUploader:
         # 定义时间戳列名
         created_col = '创建时间'
         updated_col = '更新时间'
-        current_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         
         # 复制set_typ以避免修改原始对象
         updated_set_typ = set_typ.copy()
         
-        # 添加时间戳列到set_typ
-        updated_set_typ[created_col] = 'DATETIME'
-        updated_set_typ[updated_col] = 'DATETIME'
+        # 使用MySQL的CURRENT_TIMESTAMP功能，按固定顺序添加时间戳列
+        # 创建时间：插入时自动设置，更新时不变
+        updated_set_typ[created_col] = 'TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP'
+        # 更新时间：插入和更新时都自动设置为当前时间
+        updated_set_typ[updated_col] = 'TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'
         
         # 处理DataFrame格式的数据
         if hasattr(data, 'shape') and hasattr(data, 'columns'):
             import pandas as pd
             df = data.copy()
             
-            # 移除原始数据中可能存在的时间戳列
+            # 移除原始数据中可能存在的时间戳列，让MySQL自动处理
             columns_to_remove = []
             for col in df.columns:
                 if col in [created_col, updated_col]:
                     columns_to_remove.append(col)
-                    logger.warning('移除原始数据中的时间戳列', {
-                        '库': db_name,
-                        '表': table_name,
-                        '列': col,
-                        '原因': '与自动时间戳功能冲突'
-                    })
             
             if columns_to_remove:
                 df = df.drop(columns=columns_to_remove)
             
-            # 添加时间戳列
-            df[created_col] = current_time
-            df[updated_col] = current_time
-            
+            # 不再手动添加时间戳列，让MySQL的CURRENT_TIMESTAMP自动处理
             return df, updated_set_typ
         
         # 处理字典或字典列表格式的数据
@@ -2670,10 +2728,7 @@ class MySQLUploader:
                     if key not in [created_col, updated_col]:
                         new_row[key] = value
                 
-                # 添加时间戳
-                new_row[created_col] = current_time
-                new_row[updated_col] = current_time
-                
+                # 不再手动添加时间戳，让MySQL的CURRENT_TIMESTAMP自动处理
                 processed_data.append(new_row)
             
             # 如果原始数据是单个字典，返回单个字典

{mdbq-4.1.12.dist-info → mdbq-4.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdbq
-Version: 4.1.12
+Version: 4.1.14
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.1.12.dist-info → mdbq-4.1.14.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=IFiIUlZSAqjKo9qfpz69wxp6zBBmSIhmlxQDUZwjhNE,18
+mdbq/__version__.py,sha256=zj2uAJqL04jV1wMjxAyT-2T4sjHxXiGCOfNFXJfgjz0,18
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
 mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -11,11 +11,11 @@ mdbq/log/mylogger.py,sha256=DyBftCMNLe1pTTXsa830pUtDISJxpJHFIradYtE3lFA,26418
 mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/myconf/myconf.py,sha256=x_9mS6wOfKVjCVElbruxj2yjzitbyKiTkf59quG-5Zg,32529
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=2fugLyKs_xkvYvoG0C0hRYbJ_w8-4oa1FJ_vavoD7Qo,73084
+mdbq/mysql/deduplicator.py,sha256=tzLIm9K9S0lGLlVTI0dDQVYpWX796XCuyufmw1lU26Y,73098
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=N2xHJf2CiUXjXIVBemdst-wamIP3908EGAJOFG13fCU,50475
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=7LIWclzGFWYzrt46NTJn34LJZJaz9eWiV6fIj6-ZK7k,130288
+mdbq/mysql/uploader.py,sha256=5zpvcQpa1BAtFKkVF-EQYXSieeF2hHoZQq6lmEaEnjY,133474
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.1.12.dist-info/METADATA,sha256=EI6IFZT65_tpgdTRxL9AdN5JJBKdMX4vIiMztPSKOR8,364
-mdbq-4.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mdbq-4.1.12.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.1.12.dist-info/RECORD,,
+mdbq-4.1.14.dist-info/METADATA,sha256=KTtSRhxxSKyWNWlEvHI7YIhD6XDneGbuEw3Qa1ku5EE,364
+mdbq-4.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mdbq-4.1.14.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.1.14.dist-info/RECORD,,