mdbq 3.10.2__py3-none-any.whl → 3.10.4__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.10.2'
+VERSION = '3.10.4'

mdbq/mysql/uploader.py

@@ -106,6 +106,8 @@ class MySQLUploader:
         :param connect_timeout: 连接超时(秒)，默认为10
         :param read_timeout: 读取超时(秒)，默认为30
         :param write_timeout: 写入超时(秒)，默认为30
+        :param base_excute_col: # 排重插入数据时始终排除该列
+        :param case_sensitive: # 是否保持大小写敏感，默认为False(转为小写)
         :param ssl: SSL配置字典，默认为None
         """
         self.username = username
@@ -120,6 +122,8 @@ class MySQLUploader:
         self.connect_timeout = connect_timeout
         self.read_timeout = read_timeout
         self.write_timeout = write_timeout
+        self.base_excute_col = ['id', '更新时间']  # 排重插入数据时始终排除该列
+        self.case_sensitive = False  # 是否保持大小写敏感，默认为False(转为小写)
         self.ssl = ssl
         self._prepared_statements = StatementCache(maxsize=100)
         self._max_cached_statements = 100  # 用于控制 StatementCache 类中缓存的 SQL 语句数量，最多缓存 100 条 SQL 语句
@@ -303,8 +307,8 @@ class MySQLUploader:
                     return exists
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
+                '库': db_name,
                 '检查数据库是否存在时出错': str(e),
-                '库': db_name
             })
             raise
 
@@ -377,6 +381,10 @@ class MySQLUploader:
             })
             raise ValueError(f"无效的标识符: `{identifier}`")
 
+        # 统一转为小写(除非明确要求大小写敏感)
+        if not self.case_sensitive:
+            identifier = identifier.lower()
+
         # 移除非法字符，只保留字母、数字、下划线和美元符号
         cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '_', identifier)
 
@@ -546,9 +554,9 @@ class MySQLUploader:
 
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
-                '建表失败': str(e),
                 '库': db_name,
                 '表': table_name,
+                '建表失败': str(e),
             })
             conn.rollback()
             raise
@@ -673,14 +681,14 @@ class MySQLUploader:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(sql, (db_name, table_name))
-                    set_typ = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
+                    set_typ = {row['COLUMN_NAME'].lower(): row['DATA_TYPE'] for row in cursor.fetchall()}
                     logger.debug(f"`{db_name}`.`{table_name}`: 获取表的列信息: `{set_typ}`")
                     return set_typ
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
-                '无法获取表列信息': str(e),
                 '库': db_name,
                 '表': table_name,
+                '无法获取表列信息': str(e),
             })
             raise
 
@@ -709,9 +717,9 @@ class MySQLUploader:
                                    allow_null=allow_null)
             else:
                 logger.error(sys._getframe().f_code.co_name, {
-                    '数据表不存在': table_name,
                     '库': db_name,
                     '表': table_name,
+                    '数据表不存在': table_name,
                 })
                 raise ValueError(f"数据表不存在: `{db_name}`.`{table_name}`")
 
@@ -719,9 +727,9 @@ class MySQLUploader:
         table_columns = self._get_table_columns(db_name, table_name)
         if not table_columns:
             logger.error(sys._getframe().f_code.co_name, {
-                '获取列失败': table_columns,
                 '库': db_name,
                 '表': table_name,
+                '获取列失败': table_columns,
             })
             raise ValueError(f"获取列失败 `{db_name}`.`{table_name}`")
 
@@ -729,9 +737,9 @@ class MySQLUploader:
         for col in set_typ:
             if col not in table_columns:
                 logger.error(sys._getframe().f_code.co_name, {
-                    '列不存在': f'{col} -> {table_columns}',
                     '库': db_name,
                     '表': table_name,
+                    '列不存在': f'{col} -> {table_columns}',
                 })
                 raise ValueError(f"列不存在: `{col}` -> `{db_name}`.`{table_name}`")
 
@@ -960,7 +968,7 @@ class MySQLUploader:
             默认值为 'batch'
         :raises: 可能抛出各种验证和数据库相关异常
         """
-        upload_start = time.time()
+        # upload_start = time.time()
         initial_row_count = len(data) if hasattr(data, '__len__') else 1
 
         batch_id = f"batch_{int(time.time() * 1000)}"
@@ -972,7 +980,7 @@ class MySQLUploader:
             '批次': batch_id,
             '分表方式': partition_by,
             '排重': check_duplicate,
-            '传入总计': len(data) if hasattr(data, '__len__') else 1,
+            '传入': len(data) if hasattr(data, '__len__') else 1,
             '自动建表': auto_create
         })
 
@@ -985,10 +993,10 @@ class MySQLUploader:
                 partition_by = str(partition_by).lower()
                 if partition_by not in ['year', 'month']:
                     logger.error(sys._getframe().f_code.co_name, {
-                        '分表方式必须是 "year" 或 "month" 或 "None"':  partition_by,
                         '库': db_name,
                         '表': table_name,
-                        '批次': batch_id
+                        '批次': batch_id,
+                        '分表方式必须是 "year" 或 "month" 或 "None"': partition_by,
                     })
                     raise ValueError("分表方式必须是 'year' 或 'month' 或 'None'")
 
@@ -1029,10 +1037,10 @@ class MySQLUploader:
                         partitioned_data[part_table].append(row)
                     except Exception as e:
                         logger.error(sys._getframe().f_code.co_name, {
-                            'row_data': row,
-                            '分表处理失败': str(e),
                             '库': db_name,
                             '表': table_name,
+                            'row_data': row,
+                            '分表处理失败': str(e),
                         })
                         continue  # 跳过当前行
 
@@ -1047,10 +1055,10 @@ class MySQLUploader:
                         )
                     except Exception as e:
                         logger.error(sys._getframe().f_code.co_name, {
-                            '分表': part_table,
-                            '分表上传失败': str(e),
                             '库': db_name,
                             '表': table_name,
+                            '分表': part_table,
+                            '分表上传失败': str(e),
                         })
                         continue  # 跳过当前分表，继续处理其他分表
             else:
@@ -1066,10 +1074,10 @@ class MySQLUploader:
 
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
-                '上传过程发生全局错误': str(e),
-                'error_type': type(e).__name__,
                 '库': db_name,
                 '表': table_name,
+                '上传过程发生全局错误': str(e),
+                'error_type': type(e).__name__,
             })
         finally:
             logger.info("存储完成", {
@@ -1077,7 +1085,7 @@ class MySQLUploader:
                 '表': table_name,
                 '批次': batch_id,
                 'success': success_flag,
-                '耗时': round(time.time() - upload_start, 2),
+                # '耗时': round(time.time() - upload_start, 2),
                 '数据行': initial_row_count
             })
 
@@ -1154,6 +1162,67 @@ class MySQLUploader:
             return 'batch'
         return mode.lower()
 
+    def _build_simple_insert_sql(self, db_name, table_name, columns, update_on_duplicate):
+        safe_columns = [self._validate_identifier(col) for col in columns]
+        placeholders = ','.join(['%s'] * len(safe_columns))
+
+        sql = f"""
+            INSERT INTO `{db_name}`.`{table_name}` 
+            (`{'`,`'.join(safe_columns)}`) 
+            VALUES ({placeholders})
+        """
+
+        # # 情况2：不检查重复但允许更新
+        # if update_on_duplicate:
+        #     update_clause = ", ".join([f"`{col}` = VALUES(`{col}`)"
+        #                                for col in columns])
+        #     sql += f" ON DUPLICATE KEY UPDATE {update_clause}"
+
+        return sql
+
+    def _build_duplicate_check_sql(self, db_name, table_name, all_columns,
+                                   duplicate_columns, update_on_duplicate, set_typ):
+        duplicate_columns = [_item for _item in duplicate_columns if _item.lower() not in self.base_excute_col]
+        safe_columns = [self._validate_identifier(col) for col in all_columns]
+        placeholders = ','.join(['%s'] * len(safe_columns))
+
+        # 确定排重列（排除id和更新时间列）
+        dup_cols = duplicate_columns if duplicate_columns else all_columns
+
+        # 构建排重条件
+        conditions = []
+        for col in dup_cols:
+            col_type = set_typ.get(col, '').lower()
+            if col_type.startswith('decimal'):
+                scale = self._get_decimal_scale(col_type)
+                conditions.append(f"ROUND(`{col}`, {scale}) = ROUND(%s, {scale})")
+            else:
+                conditions.append(f"`{col}` = %s")
+
+        # 情况3/5：允许更新
+        if update_on_duplicate:
+            update_clause = ", ".join([f"`{col}` = VALUES(`{col}`)"
+                                       for col in all_columns])
+            sql = f"""
+                INSERT INTO `{db_name}`.`{table_name}` 
+                (`{'`,`'.join(safe_columns)}`) 
+                VALUES ({placeholders})
+                ON DUPLICATE KEY UPDATE {update_clause}
+            """
+        else:
+            # 情况4/6：不允许更新
+            sql = f"""
+                INSERT INTO `{db_name}`.`{table_name}` 
+                (`{'`,`'.join(safe_columns)}`) 
+                SELECT {placeholders}
+                FROM DUAL
+                WHERE NOT EXISTS (
+                    SELECT 1 FROM `{db_name}`.`{table_name}`
+                    WHERE {' AND '.join(conditions)}
+                )
+            """
+        return sql
+
     def _prepare_insert_sql(
             self,
             db_name: str,
@@ -1163,55 +1232,29 @@ class MySQLUploader:
             duplicate_columns: Optional[List[str]],
             update_on_duplicate: bool
     ) -> str:
-        """准备插入SQL语句"""
-        # 获取所有列名（排除id列）
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
-        safe_columns = [self._validate_identifier(col) for col in all_columns]
-        placeholders = ','.join(['%s'] * len(safe_columns))
+        """
+        准备插入SQL语句
 
-        if check_duplicate:
-            if not duplicate_columns:
-                duplicate_columns = all_columns
-            else:
-                duplicate_columns = [col for col in duplicate_columns if col != 'id']
-
-            conditions = []
-            for col in duplicate_columns:
-                col_type = set_typ.get(col, '').lower()
-                if col_type.startswith('decimal'):
-                    scale_match = re.search(r'decimal\(\d+,(\d+)\)', col_type)
-                    scale = int(scale_match.group(1)) if scale_match else 2
-                    conditions.append(f"ROUND(`{self._validate_identifier(col)}`, {scale}) = ROUND(%s, {scale})")
-                else:
-                    conditions.append(f"`{self._validate_identifier(col)}` = %s")
-
-            where_clause = " AND ".join(conditions)
-
-            if update_on_duplicate:
-                # 更新模式 - 使用ON DUPLICATE KEY UPDATE语法
-                update_clause = ", ".join([f"`{col}` = VALUES(`{col}`)" for col in all_columns])
-                return f"""
-                    INSERT INTO `{db_name}`.`{table_name}` 
-                    (`{'`,`'.join(safe_columns)}`) 
-                    VALUES ({placeholders})
-                    ON DUPLICATE KEY UPDATE {update_clause}
-                    """
-            else:
-                return f"""INSERT INTO `{db_name}`.`{table_name}` 
-                    (`{'`,`'.join(safe_columns)}`) 
-                    SELECT {placeholders}
-                    FROM DUAL
-                    WHERE NOT EXISTS (
-                        SELECT 1 FROM `{db_name}`.`{table_name}`
-                        WHERE {where_clause}
-                    )
-                    """
-        else:
-            return f"""
-                INSERT INTO `{db_name}`.`{table_name}` 
-                (`{'`,`'.join(safe_columns)}`) 
-                VALUES ({placeholders})
-                """
+        1. 当 check_duplicate=False 时，忽略 duplicate_columns 和 update_on_duplicate 参数，直接插入全部data。
+        2. 当 check_duplicate=False 且 update_on_duplicate=True 时，由于 check_duplicate=False，直接插入全部data。
+        3. 当 check_duplicate=True 且 duplicate_columns=[] 且 update_on_duplicate=True 时，获取数据库所有列（但排除`id`和`更新时间`列），按这些列（不含`id`和`更新时间`）排重插入，遇到重复数据时更新旧数据。
+        4. 当 check_duplicate=True 且 duplicate_columns=[] 且 update_on_duplicate=False 时，获取数据库所有列（但排除`id`和`更新时间`列），按这些列（不含`id`和`更新时间`）排重插入，不考虑是否更新旧数据。
+        5. 当 check_duplicate=True 且 duplicate_columns 指定了排重列且 update_on_duplicate=True 时，按 duplicate_columns 指定的列（但排除`id`和`更新时间`）排重插入，遇到重复数据时更新旧数据。
+        6. 当 check_duplicate=True 且 duplicate_columns 指定了排重列且 update_on_duplicate=False 时，按 duplicate_columns 指定的列（但排除`id`和`更新时间`）排重插入，不考虑是否更新旧数据。
+
+        """
+        # 获取所有列名（排除id和更新时间列）
+        all_columns = [col for col in set_typ.keys()
+                       if col.lower() != 'id']
+
+        # 情况1-2：不检查重复
+        if not check_duplicate:
+            return self._build_simple_insert_sql(db_name, table_name, all_columns,
+                                                 update_on_duplicate)
+
+        # 情况3-6：检查重复
+        return self._build_duplicate_check_sql(db_name, table_name, all_columns,
+                                               duplicate_columns, update_on_duplicate, set_typ)
 
     def _execute_batch_insert(
             self,
@@ -1228,7 +1271,8 @@ class MySQLUploader:
     ) -> Tuple[int, int, int]:
         """执行批量插入操作"""
         # 获取所有列名（排除id列）
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+        all_columns = [col for col in set_typ.keys()
+                       if col.lower() != 'id']
 
         total_inserted = 0
         total_skipped = 0
@@ -1277,6 +1321,7 @@ class MySQLUploader:
             try:
                 for row_idx, row in enumerate(batch, 1):
                     result = self._process_single_row(
+                        db_name, table_name,
                         cursor, row, all_columns, sql,
                         check_duplicate, duplicate_columns
                     )
@@ -1308,6 +1353,7 @@ class MySQLUploader:
             for row_idx, row in enumerate(batch, 1):
                 try:
                     result = self._process_single_row(
+                        db_name, table_name,
                         cursor, row, all_columns, sql,
                         check_duplicate, duplicate_columns
                     )
@@ -1360,6 +1406,8 @@ class MySQLUploader:
 
     def _process_single_row(
             self,
+            db_name,
+            table_name,
             cursor,
             row: Dict,
             all_columns: List[str],
@@ -1374,6 +1422,8 @@ class MySQLUploader:
             if check_duplicate:
                 row_values += [row.get(col) for col in duplicate_columns]
 
+            # logger.info(sql)
+            # logger.info(row_values)
             cursor.execute(sql, row_values)
 
             if check_duplicate:

{mdbq-3.10.2.dist-info → mdbq-3.10.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.2
+Version: 3.10.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.10.2.dist-info → mdbq-3.10.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=tL5iFQ6j9Svg-3tbUuEZAgDFN3ipIhdJjFUPU6EHSRQ,18
+mdbq/__version__.py,sha256=L-43kDdR8o3iwkH5IR35xUFgTPugEww0j_gk9jPlkCU,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -12,7 +12,7 @@ mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
 mdbq/mysql/mysql.py,sha256=Fzaqbjg5g3HdNl50jInIrdurdzcgR2CCzdKLVImD1-Q,55339
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=XOSeGg74zN3qYFfWmLqr98H7tCj74dIMCS3C0cvS3kU,58994
+mdbq/mysql/uploader.py,sha256=ElT1-Jq5nR6qg8re0rfs26YGNPHK6zsNGc3ni7TnWFA,61954
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
-mdbq-3.10.2.dist-info/METADATA,sha256=D9d_UixDPHEbrdRE1yjA4SHjo4tYoY60_R4cBGPF3ms,364
-mdbq-3.10.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.10.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.10.2.dist-info/RECORD,,
+mdbq-3.10.4.dist-info/METADATA,sha256=z-9kwc0z6aVg7ugS9FPf2TZd1vfyNBYz0qpvsfW3b_w,364
+mdbq-3.10.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.10.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.10.4.dist-info/RECORD,,