mdbq 3.9.9__py3-none-any.whl → 3.9.11__py3-none-any.whl

mdbq/mysql/uploader.py

@@ -99,7 +99,6 @@ class MySQLUploader:
         :param port: 数据库端口，默认为3306
         :param charset: 字符集，默认为utf8mb4
         :param collation: 排序规则，默认为utf8mb4_0900_ai_ci
-
         :param max_retries: 最大重试次数，默认为10
         :param retry_interval: 重试间隔(秒)，默认为10
         :param pool_size: 连接池大小，默认为5
@@ -122,7 +121,7 @@ class MySQLUploader:
         self.write_timeout = write_timeout
         self.ssl = ssl
         self._prepared_statements = StatementCache(maxsize=100)
-        self._max_cached_statements = 100
+        self._max_cached_statements = 100  # 用于控制 StatementCache 类中缓存的 SQL 语句数量，最多缓存 100 条 SQL 语句
         self._table_metadata_cache = {}
         self.metadata_cache_ttl = 300  # 5分钟缓存时间
 
@@ -352,21 +351,29 @@ class MySQLUploader:
         :raises ValueError: 如果日期格式无效或分表方式无效
         """
         try:
-            # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
+            # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
             date_obj = self._validate_datetime(date_value, True)
         except ValueError:
-            try:
-                # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
-                date_obj = self._validate_datetime(date_value, True)
-            except ValueError:
-                error_msg = f"无效的日期格式1: {date_value}"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
+            error_msg = f"`{table_name}` 无效的日期格式1: {date_value}"
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        # try:
+        #     # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
+        #     date_obj = self._validate_datetime(date_value, True)
+        # except ValueError:
+        #     try:
+        #         # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
+        #         date_obj = self._validate_datetime(date_value, True)
+        #     except ValueError:
+        #         error_msg = f"无效的日期格式1: {date_value}"
+        #         logger.error(error_msg)
+        #         raise ValueError(error_msg)
 
         if partition_by == 'year':
             return f"{table_name}_{date_obj.year}"
         elif partition_by == 'month':
-            return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
+            return f"{table_name}_{date_obj.year}-{date_obj.month:02d}"  # 这里和兼容旧版 `2025-05`，但可能是个隐患
         else:
             error_msg = "分表方式必须是 'year' 或 'month'"
             logger.error(error_msg)
@@ -386,7 +393,11 @@ class MySQLUploader:
             raise ValueError(error_msg)
 
         # 移除非法字符，只保留字母、数字、下划线和美元符号
-        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '', identifier)
+        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '_', identifier)
+
+        # 将多个连续的下划线替换为单个下划线, 移除开头和结尾的下划线
+        cleaned = re.sub(r'_+', '_', cleaned).strip('_')
+
         if not cleaned:
             error_msg = f"无法清理异常标识符: {identifier}"
             logger.error(error_msg)
@@ -597,7 +608,7 @@ class MySQLUploader:
             column_type_lower = column_type.lower()
 
             # 处理百分比值
-            if isinstance(value, str) and '%' in value:
+            if str(value).strip().endswith('%'):
                 try:
                     # 移除百分号并转换为小数
                     percent_value = float(value.strip().replace('%', ''))
@@ -606,16 +617,22 @@ class MySQLUploader:
                 except ValueError:
                     pass  # 如果不是有效的百分比数字，继续正常处理
 
-            if 'int' in column_type_lower:
-                if isinstance(value, (str, bytes)) and not value.strip().isdigit():
-                    raise ValueError("非数字字符串无法转换为整数")
-                return int(value)
+            elif 'int' in column_type_lower:
+                if isinstance(value, str):
+                    # 移除可能的逗号和空格
+                    value = value.replace(',', '').strip()
+                    # 尝试转换为浮点数再转整数
+                    try:
+                        return int(float(value))
+                    except ValueError:
+                        raise ValueError(f"`{value}` 无法转为整数")
+                return int(value) if value is not None else None
             elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
                 if isinstance(value, str):
                     # 处理可能包含逗号的数字字符串
                     value = value.replace(',', '')
                 return float(value) if value is not None else None
-            elif '日期' in column_type_lower or 'time' in column_type_lower:
+            elif 'date' in column_type_lower or 'time' in column_type_lower:
                 if isinstance(value, (datetime.datetime, pd.Timestamp)):
                     return value.strftime('%Y-%m-%d %H:%M:%S')
                 elif isinstance(value, str):
@@ -630,12 +647,11 @@ class MySQLUploader:
                     return value.replace('\\', '\\\\').replace("'", "\\'")
                 return str(value)
             elif 'json' in column_type_lower:
-                import json
                 return json.dumps(value) if value is not None else None
             else:
                 return value
         except (ValueError, TypeError) as e:
-            error_msg = f"数据类型转换异常 {value} to type {column_type}: {str(e)}"
+            error_msg = f"转换异常 -> 无法将 `{value}` 的数据类型转为: `{column_type}` -> {str(e)}"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
@@ -681,7 +697,8 @@ class MySQLUploader:
             auto_create: bool,
             date_column: Optional[str],
             indexes: Optional[List[str]],
-            batch_id: Optional[str] = None
+            batch_id: Optional[str] = None,
+            update_on_duplicate: bool = False
     ):
         """实际执行表上传的方法"""
         # 检查表是否存在
@@ -711,7 +728,9 @@ class MySQLUploader:
         # 插入数据
         self._insert_data(
             db_name, table_name, data, set_typ,
-            check_duplicate, duplicate_columns
+            check_duplicate, duplicate_columns,
+            batch_id=batch_id,
+            update_on_duplicate=update_on_duplicate
         )
 
     def _infer_data_type(self, value: Any) -> str:
@@ -721,12 +740,13 @@ class MySQLUploader:
         :param value: 要推断的值
         :return: MySQL数据类型字符串
         """
-        if value is None:
+        if value is None or str(value).lower() in ['', 'none', 'nan']:
             return 'VARCHAR(255)'  # 默认字符串类型
 
         # 检查是否是百分比字符串
-        if isinstance(value, str) and '%' in value:
-            return 'DECIMAL(10,4)'  # 百分比统一使用DECIMAL(10,4)
+        if isinstance(value, str):
+            if value.endswith('%'):
+                return 'DECIMAL(10,4)'  # 百分比统一使用DECIMAL(10,4)
 
         if isinstance(value, bool):
             return 'TINYINT(1)'
@@ -773,6 +793,26 @@ class MySQLUploader:
         else:
             return 'VARCHAR(255)'
 
+    def normalize_column_names(self, data: Union[pd.DataFrame, List[Dict[str, Any]]]) -> Union[
+        pd.DataFrame, List[Dict[str, Any]]]:
+        """
+        1. pandas：规范化列名
+        2. 字典列表：规范化每个字典的键
+
+        参数：
+            data: 输入数据，支持两种类型：
+                  - pandas.DataFrame：将规范化其列名
+                  - List[Dict[str, Any]]：将规范化列表中每个字典的键
+        """
+        if isinstance(data, pd.DataFrame):
+            # 处理DataFrame
+            data.columns = [self._validate_identifier(col) for col in data.columns]
+            return data
+        elif isinstance(data, list):
+            # 处理字典列表
+            return [{self._validate_identifier(k): v for k, v in item.items()} for item in data]
+        return data
+
     def _prepare_data(
             self,
             data: Union[Dict, List[Dict], pd.DataFrame],
@@ -807,6 +847,9 @@ class MySQLUploader:
             logger.error(error_msg)
             raise ValueError(error_msg)
 
+        # 统一处理原始数据中列名的特殊字符
+        data = self.normalize_column_names(data)
+
         # 将set_typ的键转为小写
         set_typ = {k.lower(): v for k, v in set_typ.items()}
 
@@ -826,11 +869,11 @@ class MySQLUploader:
                 if sample_values:
                     inferred_type = self._infer_data_type(sample_values[0])
                     filtered_set_typ[col] = inferred_type
-                    logger.debug(f"自动推断列'{col}'的数据类型为: {inferred_type}")
+                    logger.debug(f"自动推断列 `{col}` 的数据类型为: {inferred_type}")
                 else:
                     # 没有样本值，使用默认类型
                     filtered_set_typ[col] = 'VARCHAR(255)'
-                    logger.debug(f"为列'{col}'使用默认数据类型: VARCHAR(255)")
+                    logger.debug(f"列 `{col}` 使用默认数据类型: VARCHAR(255)")
 
         prepared_data = []
         for row_idx, row in enumerate(data, 1):
@@ -842,7 +885,7 @@ class MySQLUploader:
 
                 if col_name not in row:
                     if not allow_null:
-                        error_msg = f"Row {row_idx}: Missing required column '{col_name}' in data"
+                        error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`"
                         logger.error(error_msg)
                         raise ValueError(error_msg)
                     prepared_row[col_name] = None
@@ -850,7 +893,7 @@ class MySQLUploader:
                     try:
                         prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null)
                     except ValueError as e:
-                        error_msg = f"Row {row_idx}, column '{col_name}': {str(e)}"
+                        error_msg = f"行号:{row_idx}, 列名:`{col_name}`-> 报错: {str(e)}"
                         logger.error(error_msg)
                         raise ValueError(error_msg)
             prepared_data.append(prepared_row)
@@ -871,7 +914,8 @@ class MySQLUploader:
             partition_by: Optional[str] = None,
             partition_date_column: str = '日期',
             auto_create: bool = True,
-            indexes: Optional[List[str]] = None
+            indexes: Optional[List[str]] = None,
+            update_on_duplicate: bool = False
     ):
         """
         上传数据到数据库的主入口方法
@@ -888,6 +932,7 @@ class MySQLUploader:
         :param partition_date_column: 用于分表的日期列名，默认为'日期'
         :param auto_create: 表不存在时是否自动创建，默认为True
         :param indexes: 需要创建索引的列列表，可选
+        :param update_on_duplicate: 遇到重复数据时是否更新旧数据(默认为False)
         :raises: 可能抛出各种验证和数据库相关异常
         """
         upload_start = time.time()
@@ -898,25 +943,25 @@ class MySQLUploader:
 
         logger.info("开始上传数据", {
             '批次号': batch_id,
-            'database': db_name,
-            'table': table_name,
+            '库': db_name,
+            '表': table_name,
             '分表方式': partition_by,
-            '是否排重': check_duplicate,
+            '排重': check_duplicate,
             '总计行数': len(data) if hasattr(data, '__len__') else 1,
             '自动建表': auto_create
         })
 
         try:
-            # # 验证参数
-            # if not set_typ:
-            #     error_msg = "列的数据类型缺失"
-            #     logger.error(error_msg)
-            #     raise ValueError(error_msg)
-
-            if partition_by and partition_by not in ['year', 'month']:
-                error_msg = "分表方式必须是 'year' 或 'month'"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
+            # 验证参数
+            if not set_typ:
+                logger.debug(f'set_typ 参数缺失，建表不指定数据类型字典，后续存储数据容易引发异常')
+
+            if partition_by:
+                partition_by = str(partition_by).lower()
+                if partition_by not in ['year', 'month']:
+                    error_msg = "分表方式必须是 'year' 或 'month'"
+                    logger.error(error_msg)
+                    raise ValueError(error_msg)
 
             # 准备数据
             prepared_data, filtered_set_typ = self._prepare_data(data, set_typ, allow_null)
@@ -962,7 +1007,7 @@ class MySQLUploader:
                             db_name, part_table, part_data, filtered_set_typ,
                             primary_keys, check_duplicate, duplicate_columns,
                             allow_null, auto_create, partition_date_column,
-                            indexes, batch_id
+                            indexes, batch_id, update_on_duplicate
                         )
                     except Exception as e:
                         logger.error("分表上传失败", {
@@ -976,7 +1021,7 @@ class MySQLUploader:
                     db_name, table_name, prepared_data, filtered_set_typ,
                     primary_keys, check_duplicate, duplicate_columns,
                     allow_null, auto_create, partition_date_column,
-                    indexes, batch_id
+                    indexes, batch_id, update_on_duplicate
                 )
 
             success_flag = True
@@ -1004,7 +1049,8 @@ class MySQLUploader:
             check_duplicate: bool = False,
             duplicate_columns: Optional[List[str]] = None,
             batch_size: int = 1000,
-            batch_id: Optional[str] = None
+            batch_id: Optional[str] = None,
+            update_on_duplicate: bool = False
     ):
         """
         实际执行数据插入的方法
@@ -1016,6 +1062,7 @@ class MySQLUploader:
         :param check_duplicate: 是否检查重复数据，默认为False
         :param duplicate_columns: 用于检查重复的列，可选
         :param batch_size: 批量插入大小，默认为1000
+        :param update_on_duplicate: 遇到重复数据时是否更新旧数据(默认为False)
         :param batch_id: 批次ID用于日志追踪，可选
         """
         if not data:
@@ -1048,8 +1095,21 @@ class MySQLUploader:
 
             where_clause = " AND ".join(conditions)
 
-            sql = f"""
+            if update_on_duplicate:
+                # 更新模式 - 使用ON DUPLICATE KEY UPDATE语法
+                update_clause = ", ".join([f"`{col}` = VALUES(`{col}`)" for col in all_columns])
+                sql = f"""
                     INSERT INTO `{db_name}`.`{table_name}` 
+                    (`{'`,`'.join(safe_columns)}`) 
+                    VALUES ({placeholders})
+                    ON DUPLICATE KEY UPDATE {update_clause}
+                    """
+
+                # 注意：在update_on_duplicate模式下，row_values只需要插入数据，不需要排重列值
+                def prepare_values(row):
+                    return [row.get(col) for col in all_columns]
+            else:
+                sql = f"""INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
                     SELECT {placeholders}
                     FROM DUAL
@@ -1058,6 +1118,10 @@ class MySQLUploader:
                         WHERE {where_clause}
                     )
                     """
+
+                # 在check_duplicate模式下，row_values需要插入数据+排重列值
+                def prepare_values(row):
+                    return [row.get(col) for col in all_columns] + [row.get(col) for col in duplicate_columns]
         else:
             sql = f"""
                 INSERT INTO `{db_name}`.`{table_name}` 
@@ -1065,6 +1129,10 @@ class MySQLUploader:
                 VALUES ({placeholders})
                 """
 
+            # 普通模式下，row_values只需要插入数据
+            def prepare_values(row):
+                return [row.get(col) for col in all_columns]
+
         total_inserted = 0
         total_skipped = 0
         total_failed = 0  # 失败计数器
@@ -1080,11 +1148,7 @@ class MySQLUploader:
                     for row in batch:
                         try:
                             # 准备参数
-                            row_values = [row.get(col) for col in all_columns]
-                            # 如果是排重检查，添加排重列值
-                            if check_duplicate:
-                                row_values += [row.get(col) for col in duplicate_columns]
-
+                            row_values = prepare_values(row)
                             cursor.execute(sql, row_values)
                             successful_rows += 1
                             conn.commit()  # 每次成功插入后提交
@@ -1096,13 +1160,13 @@ class MySQLUploader:
                             # 记录失败行详细信息
                             error_details = {
                                 '批次号': batch_id,
-                                'database': db_name,
-                                'table': table_name,
+                                '库': db_name,
+                                '表': table_name,
                                 'error_type': type(e).__name__,
                                 'error_message': str(e),
-                                'column_types': set_typ,
+                                '数据类型': set_typ,
                                 '是否排重': check_duplicate,
-                                'duplicate_columns': duplicate_columns
+                                '排重列': duplicate_columns
                             }
                             logger.error(f"单行插入失败: {error_details}")
                             continue  # 跳过当前行，继续处理下一行
@@ -1227,7 +1291,7 @@ def main():
     # 准备数据
     data = [
         {'日期': '2023-01-8', 'name': 'JACk', 'AGE': '24', 'salary': 555.1545},
-        {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 35, 'salary': 100},
+        {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 35, 'salary': '100'},
         {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 30, 'salary': 0.0},
         {'日期': '2023-02-20', 'name': 'Bob', 'AGE': 25, 'salary': 45000.75}
     ]

{mdbq-3.9.9.dist-info → mdbq-3.9.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.9
+Version: 3.9.11
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.9.9.dist-info → mdbq-3.9.11.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=Z38j4uvZuqpFYiUEq0FTd82-1Y90RoVwpNEDWVHNTkk,17
+mdbq/__version__.py,sha256=zVFC_9Hi22FnOaPTZtjaH6okUJz6Vi_VZ5smgBB5eZQ,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -10,9 +10,9 @@ mdbq/log/mylogger.py,sha256=jHCVO7KPQrg2kcCaIrakHivZmFBJyy-24sIn2rsbK4Y,24440
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
-mdbq/mysql/mysql.py,sha256=jTcizvUtRdwMhWK2i_LA9yDPmcifLjUzVhwTbC3wfJk,119785
+mdbq/mysql/mysql.py,sha256=0Yaaw178wk_iRxI2PpVGeec_tDal4Fq5jyCyB0scgvQ,55339
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=mIgUnV7MwIkrbG-dchMkMzWo_N-XrQROLWTGGGuD_ts,49171
+mdbq/mysql/uploader.py,sha256=_Z0m7nEXzmxY6josxBeQR7nG-bsL8FQHR_RDLjj7IzE,52480
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
-mdbq-3.9.9.dist-info/METADATA,sha256=F6RAyI8aGmpT-VLwVeY7jw13qemIce-PMH2Ri335GAE,363
-mdbq-3.9.9.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.9.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.9.9.dist-info/RECORD,,
+mdbq-3.9.11.dist-info/METADATA,sha256=NuMxnqS57k-Z2opqI-yPWqkFPdTJqPJUaqZSwUDb1uY,364
+mdbq-3.9.11.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.11.dist-info/RECORD,,