mdbq 3.11.1__py3-none-any.whl → 3.11.3__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.11.1'
+VERSION = '3.11.3'

mdbq/aggregation/query_data.py

@@ -9,14 +9,11 @@ import pandas as pd
 import numpy as np
 from functools import wraps
 import platform
-import json
 import os
 import time
 import calendar
 import concurrent.futures
-import traceback
 import logging
-import sys
 
 """
 

mdbq/log/mylogger.py

@@ -7,7 +7,7 @@ import sys
 import time
 import threading
 import queue
-from typing import Optional, Dict, Any, List, Callable, Union
+from typing import Optional, Dict, Any, List, Callable
 import atexit
 import traceback
 import inspect

mdbq/mysql/deduplicator.py

@@ -18,7 +18,7 @@ from datetime import datetime
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
     name='deduplicator',
-    logging_mode='both',
+    logging_mode='file',
     log_level='info',
     log_file='deduplicator.log',
     log_format='json',

mdbq/mysql/mysql.py

@@ -7,10 +7,9 @@ import warnings
 import pymysql
 import pandas as pd
 from sqlalchemy import create_engine
-import os
 from mdbq.other import otk
 from mdbq.log import mylogger
-import json
+import math
 
 warnings.filterwarnings('ignore')
 """
@@ -131,7 +130,7 @@ class MysqlUpload:
         new_dict_data: dict = {}
         for k, v in dict_data.items():
             k = str(k).lower()
-            k = re.sub(r'[()\-，,$&~^、 （）\"\'“”=·/。》《><！!`]', '_', k, re.IGNORECASE)
+            k = re.sub(r'[()\-，,$&~^、 （）\"\'"="·/。》《><！!`]', '_', k, re.IGNORECASE)
             k = k.replace('）', '')
             k = re.sub(r'_{2,}', '_', k)
             k = re.sub(r'_+$', '', k)
@@ -526,7 +525,7 @@ class MysqlUpload:
         new_dict_data = {}
         for k, v in dict_data.items():
             k = str(k).lower()
-            k = re.sub(r'[()\-，,$&~^、 （）\"\'“”=·/。》《><！!`]', '_', k, re.IGNORECASE)
+            k = re.sub(r'[()\-，,$&~^、 （）\"\'"="·/。》《><！!`]', '_', k, re.IGNORECASE)
             k = k.replace('）', '')
             k = re.sub(r'_{2,}', '_', k)
             k = re.sub(r'_+$', '', k)

mdbq/mysql/uploader.py

@@ -10,8 +10,8 @@ from mdbq.log import mylogger
 from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
 import json
-from collections import OrderedDict
 import sys
+from decimal import Decimal, InvalidOperation
 
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
@@ -28,62 +28,44 @@ logger = mylogger.MyLogger(
 )
 
 
-def count_decimal_places(num_str):
+def count_decimal_places(num_str: str) -> Tuple[int, int]:
     """
-    计算数字字符串的小数位数，支持科学计数法
-
-    :param num_str: 数字字符串
-    :return: 返回元组(整数位数, 小数位数)
-    :raises: 无显式抛出异常，但正则匹配失败时返回(0, 0)
-    """
-    match = re.match(r'^[-+]?\d+(\.\d+)?([eE][-+]?\d+)?$', str(num_str))
-    if match:
-        # 如果是科学计数法
-        match = re.findall(r'(\d+)\.(\d+)[eE][-+]?(\d+)$', str(num_str))
-        if match:
-            if len(match[0]) == 3:
-                if int(match[0][2]) < len(match[0][1]):
-                    # count_int 清除整数部分开头的 0 并计算整数位数
-                    count_int = len(re.sub('^0+', '', str(match[0][0]))) + int(match[0][2])
-                    # 计算小数位数
-                    count_float = len(match[0][1]) - int(match[0][2])
-                    return count_int, count_float
-        # 如果是普通小数
-        match = re.findall(r'(\d+)\.(\d+)$', str(num_str))
-        if match:
-            count_int = len(re.sub('^0+', '', str(match[0][0])))
-            count_float = len(match[0][1])
-            return count_int, count_float  # 计算小数位数
-    return 0, 0
-
-
-class StatementCache(OrderedDict):
-    """
-    基于OrderedDict实现的LRU缓存策略，用于缓存SQL语句
-    
-    这个类继承自OrderedDict，实现了最近最少使用(LRU)的缓存策略。
-    当缓存达到最大容量时，会自动删除最早添加的项。
+    统计小数点前后位数，支持科学计数法。
+    返回：(整数位数, 小数位数)
     """
+    try:
+        d = Decimal(str(num_str))
+        sign, digits, exponent = d.as_tuple()
+        int_part = len(digits) + exponent if exponent < 0 else len(digits)
+        dec_part = -exponent if exponent < 0 else 0
+        return max(int_part, 0), max(dec_part, 0)
+    except (InvalidOperation, ValueError, TypeError):
+        return (0, 0)
+
+
+class StatementCache(dict):
+    """简单LRU缓存实现，用于SQL语句缓存"""
     def __init__(self, maxsize=100):
-        """
-        初始化缓存
-
-        :param maxsize: 最大缓存大小，默认为100条SQL语句
-        """
         super().__init__()
-        self.maxsize = maxsize
-
+        self._maxsize = maxsize
+        self._order = []
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+        self._order.remove(key)
+        self._order.append(key)
+        return value
     def __setitem__(self, key, value):
-        """
-        重写设置项方法，实现LRU策略
-
-        :param key: 缓存键
-        :param value: 缓存值
-        """
+        if key in self:
+            self._order.remove(key)
+        elif len(self._order) >= self._maxsize:
+            oldest = self._order.pop(0)
+            super().__delitem__(oldest)
         super().__setitem__(key, value)
-        if len(self) > self.maxsize:
-            self.popitem(last=False)
-
+        self._order.append(key)
+    def get(self, key, default=None):
+        if key in self:
+            return self[key]
+        return default
 
 class MySQLUploader:
     """
@@ -194,22 +176,22 @@ class MySQLUploader:
             logger.error('连接池创建失败', {'error': str(e), 'host': self.host, 'port': self.port})
             raise ConnectionError(f'连接池创建失败: {str(e)}')
 
-    def _execute_with_retry(self, func):
+    @staticmethod
+    def _execute_with_retry(func):
         """
         带重试机制的装饰器，用于数据库操作
-
         :param func: 被装饰的函数
         :return: 装饰后的函数
         :raises: 可能抛出原始异常或最后一次重试的异常
         """
         @wraps(func)
-        def wrapper(*args, **kwargs):
+        def wrapper(self, *args, **kwargs):
             last_exception = None
             operation = func.__name__
             logger.debug(f'开始执行操作: {operation}', {'max_retries': self.max_retries})
             for attempt in range(self.max_retries):
                 try:
-                    result = func(*args, **kwargs)
+                    result = func(self, *args, **kwargs)
                     if attempt > 0:
                         logger.info('操作成功(重试后)', {'operation': operation, 'attempts': attempt + 1})
                     else:
@@ -255,6 +237,7 @@ class MySQLUploader:
             raise last_exception if last_exception else Exception('发生未知错误')
         return wrapper
 
+    @_execute_with_retry
     def _get_connection(self) -> pymysql.connections.Connection:
         """
         从连接池获取数据库连接
@@ -270,6 +253,7 @@ class MySQLUploader:
             logger.error('获取数据库连接失败', {'error': str(e)})
             raise ConnectionError(f'连接数据库失败: {str(e)}')
 
+    @_execute_with_retry
     def _check_database_exists(self, db_name: str) -> bool:
         """
         检查数据库是否存在
@@ -280,6 +264,7 @@ class MySQLUploader:
         """
         db_name = self._validate_identifier(db_name)
         sql = 'SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s'
+        conn = None
         try:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
@@ -291,6 +276,7 @@ class MySQLUploader:
             logger.error('检查数据库是否存在时出错', {'库': db_name, '错误': str(e)})
             raise
 
+    @_execute_with_retry
     def _create_database(self, db_name: str) -> None:
         """
         创建数据库
@@ -300,6 +286,7 @@ class MySQLUploader:
         """
         db_name = self._validate_identifier(db_name)
         sql = f'CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}'
+        conn = None
         try:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
@@ -308,7 +295,8 @@ class MySQLUploader:
                 logger.info('数据库已创建', {'库': db_name})
         except Exception as e:
             logger.error('无法创建数据库', {'库': db_name, '错误': str(e)})
-            conn.rollback()
+            if conn is not None:
+                conn.rollback()
             raise
 
     def _get_partition_table_name(self, table_name: str, date_value: str, partition_by: str) -> str:
@@ -345,10 +333,8 @@ class MySQLUploader:
         if not identifier or not isinstance(identifier, str):
             logger.error('无效的标识符', {'标识符': identifier})
             raise ValueError(f"无效的标识符: `{identifier}`")
-        if not self.case_sensitive:
-            cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '_', identifier)
-        else:
-            cleaned = identifier
+        # 始终做特殊字符清理
+        cleaned = re.sub(r'[^-\uFFFF\w\u4e00-\u9fff$]', '_', identifier)
         cleaned = re.sub(r'_+', '_', cleaned).strip('_')
         if not cleaned:
             logger.error('无法清理异常标识符', {'原始标识符': identifier})
@@ -362,6 +348,7 @@ class MySQLUploader:
             return f"`{cleaned}`"
         return cleaned
 
+    @_execute_with_retry
     def _check_table_exists(self, db_name: str, table_name: str) -> bool:
         """
         检查表是否存在
@@ -396,6 +383,7 @@ class MySQLUploader:
         logger.debug('表存在检查', {'库': db_name, '表': table_name, '存在': result})
         return result
 
+    @_execute_with_retry
     def _create_table(
             self,
             db_name: str,
@@ -407,16 +395,7 @@ class MySQLUploader:
             allow_null: bool = False
     ) -> None:
         """
-        创建数据表
-
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param set_typ: 列名和数据类型字典 {列名: 数据类型}
-        :param primary_keys: 主键列列表，可选
-        :param date_column: 日期列名，可选，如果存在将设置为索引
-        :param indexes: 需要创建索引的列列表，可选
-        :param allow_null: 是否允许空值，默认为False
-        :raises: 可能抛出数据库相关异常
+        创建数据表，优化索引创建方式
         """
         db_name = self._validate_identifier(db_name)
         table_name = self._validate_identifier(table_name)
@@ -439,40 +418,35 @@ class MySQLUploader:
             primary_keys = ['id']
         safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
         primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+        # 索引统一在CREATE TABLE中定义
+        index_defs = []
+        if date_column and date_column in set_typ:
+            safe_date_col = self._validate_identifier(date_column)
+            index_defs.append(f"INDEX `idx_{safe_date_col}` (`{safe_date_col}`)")
+        if indexes:
+            for idx_col in indexes:
+                if idx_col in set_typ:
+                    safe_idx_col = self._validate_identifier(idx_col)
+                    index_defs.append(f"INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)")
+        index_sql = (',' + ','.join(index_defs)) if index_defs else ''
         sql = f"""
         CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
             {','.join(column_defs)}
             {primary_key_sql}
+            {index_sql}
         ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
         """
+        conn = None
         try:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
-                    logger.info('数据表已创建', {'库': db_name, '表': table_name})
-                index_statements = []
-                if date_column and date_column in set_typ:
-                    safe_date_col = self._validate_identifier(date_column)
-                    index_statements.append(
-                        f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
-                    )
-                if indexes:
-                    for idx_col in indexes:
-                        if idx_col in set_typ:
-                            safe_idx_col = self._validate_identifier(idx_col)
-                            index_statements.append(
-                                f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)"
-                            )
-                if index_statements:
-                    with conn.cursor() as cursor:
-                        for stmt in index_statements:
-                            cursor.execute(stmt)
-                            logger.debug('执行索引语句', {'SQL': stmt})
                 conn.commit()
-                logger.info('索引已添加', {'库': db_name, '表': table_name, '索引': indexes})
+                logger.info('数据表及索引已创建', {'库': db_name, '表': table_name, '索引': indexes})
         except Exception as e:
             logger.error('建表失败', {'库': db_name, '表': table_name, '错误': str(e)})
-            conn.rollback()
+            if conn is not None:
+                conn.rollback()
             raise
 
     def _validate_datetime(self, value: str, date_type: bool = False) -> Any:
@@ -511,19 +485,24 @@ class MySQLUploader:
         logger.error('无效的日期格式', {'值': value})
         raise ValueError(f"无效的日期格式: `{value}`")
 
-    def _validate_value(self, value: Any, column_type: str, allow_null: bool) -> Any:
+    def _validate_value(self, value: Any, column_type: str, allow_null: bool, db_name: str = None, table_name: str = None, col_name: str = None) -> Any:
         """
         根据列类型验证并转换数据值
 
         :param value: 要验证的值
         :param column_type: 列的数据类型
         :param allow_null: 是否允许空值
+        :param db_name: 数据库名（用于日志）
+        :param table_name: 表名（用于日志）
+        :param col_name: 列名（用于日志）
         :return: 转换后的值
         :raises ValueError: 当值转换失败时抛出
         """
         if value is None:
             if not allow_null:
-                logger.warning('字段值为None但不允许空值', {'字段类型': column_type})
+                logger.warning('字段值为None但不允许空值, 已填充为none', {
+                    '库': db_name, '表': table_name, '列': col_name, '字段类型': column_type
+                })
                 return 'none'
             return None
         try:
@@ -536,14 +515,18 @@ class MySQLUploader:
                     logger.debug('百分比字符串转小数', {'原始': value, '结果': decimal_value})
                     return decimal_value
                 except ValueError:
-                    logger.warning('百分比字符串转小数失败', {'原始': value})
+                    logger.warning('百分比字符串转小数失败', {
+                        '库': db_name, '表': table_name, '列': col_name, '原始': value
+                    })
             elif 'int' in column_type_lower:
                 if isinstance(value, str):
                     value = value.replace(',', '').strip()
                     try:
                         return int(float(value))
                     except ValueError:
-                        logger.error('字符串转整数失败', {'值': value})
+                        logger.error('字符串转整数失败', {
+                            '库': db_name, '表': table_name, '列': col_name, '值': value
+                        })
                         raise ValueError(f"`{value}` -> 无法转为整数")
                 return int(value) if value is not None else None
             elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
@@ -557,7 +540,9 @@ class MySQLUploader:
                     try:
                         return self._validate_datetime(value)
                     except ValueError as e:
-                        logger.error('无效日期格式', {'值': value, '错误': str(e)})
+                        logger.error('无效日期格式', {
+                            '库': db_name, '表': table_name, '列': col_name, '值': value, '错误': str(e)
+                        })
                         raise ValueError(f"无效日期格式: `{value}` -> {str(e)}")
                 return str(value)
             elif 'char' in column_type_lower or 'text' in column_type_lower:
@@ -569,9 +554,12 @@ class MySQLUploader:
             else:
                 return value
         except (ValueError, TypeError) as e:
-            logger.error('数据类型转换异常', {'值': value, '目标类型': column_type, '错误': str(e)})
+            logger.error('数据类型转换异常', {
+                '库': db_name, '表': table_name, '列': col_name, '值': value, '目标类型': column_type, '错误': str(e)
+            })
             raise ValueError(f"转换异常 -> 无法将 `{value}` 的数据类型转为: `{column_type}` -> {str(e)}")
 
+    @_execute_with_retry
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
         """
         获取表的列名和数据类型
@@ -593,7 +581,10 @@ class MySQLUploader:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(sql, (db_name, table_name))
-                    set_typ = {row['COLUMN_NAME'].lower(): row['DATA_TYPE'] for row in cursor.fetchall()}
+                    if self.case_sensitive:
+                        set_typ = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
+                    else:
+                        set_typ = {row['COLUMN_NAME'].lower(): row['DATA_TYPE'] for row in cursor.fetchall()}
                     logger.debug('获取表的列信息', {'库': db_name, '表': table_name, '列信息': set_typ})
                     return set_typ
         except Exception as e:
@@ -727,39 +718,38 @@ class MySQLUploader:
         """
         1. pandas：规范化列名
         2. 字典列表：规范化每个字典的键
-
-        参数：
-            data: 输入数据，支持两种类型：
-                  - pandas.DataFrame：将规范化其列名
-                  - List[Dict[str, Any]]：将规范化列表中每个字典的键
         """
         if isinstance(data, pd.DataFrame):
-            data.columns = [self._validate_identifier(col) for col in data.columns]
+            if self.case_sensitive:
+                data.columns = [self._validate_identifier(col) for col in data.columns]
+            else:
+                data.columns = [self._validate_identifier(col).lower() for col in data.columns]
             return data
         elif isinstance(data, list):
-            return [{self._validate_identifier(k): v for k, v in item.items()} for item in data]
+            if self.case_sensitive:
+                return [{self._validate_identifier(k): v for k, v in item.items()} for item in data]
+            else:
+                return [{self._validate_identifier(k).lower(): v for k, v in item.items()} for item in data]
         return data
 
     def _prepare_data(
             self,
             data: Union[Dict, List[Dict], pd.DataFrame],
             set_typ: Dict[str, str],
-            allow_null: bool = False
+            allow_null: bool = False,
+            db_name: str = None, 
+            table_name: str = None, 
     ) -> Tuple[List[Dict], Dict[str, str]]:
         """
         准备要上传的数据，验证并转换数据类型
-
-        :param data: 输入数据，可以是字典、字典列表或DataFrame
-        :param set_typ: 列名和数据类型字典 {列名: 数据类型}
-        :param allow_null: 是否允许空值
-        :return: 元组(准备好的数据列表, 过滤后的列类型字典)
-        :raises ValueError: 当数据验证失败时抛出
         """
         # 统一数据格式为字典列表
         if isinstance(data, pd.DataFrame):
             try:
-                # 将列名转为小写
-                data.columns = [col.lower() for col in data.columns]
+                if self.case_sensitive:
+                    data.columns = [col for col in data.columns]
+                else:
+                    data.columns = [col.lower() for col in data.columns]
                 data = data.replace({pd.NA: None}).to_dict('records')
             except Exception as e:
                 logger.error('数据转字典时发生错误', {
@@ -769,10 +759,15 @@ class MySQLUploader:
                 })
                 raise ValueError(f"数据转字典时发生错误: {e}")
         elif isinstance(data, dict):
-            data = [{k.lower(): v for k, v in data.items()}]
+            if self.case_sensitive:
+                data = [{k: v for k, v in data.items()}]
+            else:
+                data = [{k.lower(): v for k, v in data.items()}]
         elif isinstance(data, list) and all(isinstance(item, dict) for item in data):
-            # 将列表中的每个字典键转为小写
-            data = [{k.lower(): v for k, v in item.items()} for item in data]
+            if self.case_sensitive:
+                data = [{k: v for k, v in item.items()} for item in data]
+            else:
+                data = [{k.lower(): v for k, v in item.items()} for item in data]
         else:
             logger.error('数据结构必须是字典、列表、字典列表或dataframe', {
                 'data': self._shorten_for_log(data),
@@ -783,8 +778,11 @@ class MySQLUploader:
         # 统一处理原始数据中列名的特殊字符
         data = self.normalize_column_names(data)
 
-        # 将set_typ的键转为小写
-        set_typ = {k.lower(): v for k, v in set_typ.items()}
+        # set_typ的键处理
+        if self.case_sensitive:
+            set_typ = {k: v for k, v in set_typ.items()}
+        else:
+            set_typ = {k.lower(): v for k, v in set_typ.items()}
 
         # 获取数据中实际存在的列名
         data_columns = set()
@@ -797,25 +795,25 @@ class MySQLUploader:
             if col in set_typ:
                 filtered_set_typ[col] = set_typ[col]
             else:
-                # 如果列不在set_typ中，尝试推断类型
-                sample_values = [row[col] for row in data if col in row and row[col] is not None][:10]
-                if sample_values:
-                    inferred_type = self._infer_data_type(sample_values[0])
-                    filtered_set_typ[col] = inferred_type
-                    logger.debug(f"自动推断列 `{col}` 的数据类型为: `{inferred_type}`")
-                else:
-                    # 没有样本值，使用默认类型
-                    filtered_set_typ[col] = 'VARCHAR(255)'
-                    logger.debug(f"列 `{col}` 使用默认数据类型: VARCHAR(255)")
+                # 如果列不在set_typ中，采样多个非None值推断类型
+                sample_values = [row[col] for row in data if col in row and row[col] is not None][:5]
+                inferred_type = None
+                for val in sample_values:
+                    inferred_type = self._infer_data_type(val)
+                    if inferred_type:
+                        break
+                if not inferred_type:
+                    inferred_type = 'VARCHAR(255)'
+                filtered_set_typ[col] = inferred_type
+                logger.debug(f"自动推断列 `{col}` 的数据类型为: `{inferred_type}`")
 
         prepared_data = []
         for row_idx, row in enumerate(data, 1):
             prepared_row = {}
             for col_name in filtered_set_typ:
                 # 跳过id列，不允许外部传入id
-                if col_name.lower() == 'id':
+                if (self.case_sensitive and col_name == 'id') or (not self.case_sensitive and col_name.lower() == 'id'):
                     continue
-
                 if col_name not in row:
                     if not allow_null:
                         error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`"
@@ -824,7 +822,7 @@ class MySQLUploader:
                     prepared_row[col_name] = None
                 else:
                     try:
-                        prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null)
+                        prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
                     except ValueError as e:
                         logger.error('数据验证失败', {
                             '列': col_name,
@@ -856,7 +854,7 @@ class MySQLUploader:
             transaction_mode: str = "batch"
     ):
         """
-        上传数据到数据库的主入口方法
+        上传数据到数据库的主入口方法，分表逻辑异常处理统计丢弃数据
 
         :param db_name: 数据库名
         :param table_name: 表名
@@ -882,6 +880,7 @@ class MySQLUploader:
 
         batch_id = f"batch_{int(time.time() * 1000)}"
         success_flag = False
+        dropped_rows = 0
 
         logger.info("开始上传", {
             '库': db_name,
@@ -918,7 +917,7 @@ class MySQLUploader:
                     raise ValueError("分表方式必须是 'year' 或 'month' 或 'None'")
 
             # 准备数据
-            prepared_data, filtered_set_typ = self._prepare_data(data, set_typ, allow_null)
+            prepared_data, filtered_set_typ = self._prepare_data(data, set_typ, allow_null, db_name, table_name)
 
             # 检查数据库是否存在
             if not self._check_database_exists(db_name):
@@ -945,8 +944,8 @@ class MySQLUploader:
                                 'row': self._shorten_for_log(row),
                                 'func': sys._getframe().f_code.co_name,
                             })
-                            continue  # 跳过当前行
-
+                            dropped_rows += 1
+                            continue
                         part_table = self._get_partition_table_name(
                             table_name,
                             str(row[partition_date_column]),
@@ -963,7 +962,8 @@ class MySQLUploader:
                             'error': str(e),
                             'func': sys._getframe().f_code.co_name,
                         })
-                        continue  # 跳过当前行
+                        dropped_rows += 1
+                        continue
 
                 # 对每个分表执行上传
                 for part_table, part_data in partitioned_data.items():
@@ -1010,10 +1010,11 @@ class MySQLUploader:
                 '表': table_name,
                 '批次': batch_id,
                 'finish': success_flag,
-                # '耗时': round(time.time() - upload_start, 2),
-                '数据行': initial_row_count
+                '数据行': initial_row_count,
+                '丢弃行数': dropped_rows
             })
 
+    @_execute_with_retry
     def _insert_data(
             self,
             db_name: str,
@@ -1178,34 +1179,27 @@ class MySQLUploader:
             update_on_duplicate: bool
     ) -> str:
         """
-        准备插入SQL语句
-
-        1. 当 check_duplicate=False 时，忽略 duplicate_columns 和 update_on_duplicate 参数，直接插入全部data。
-        2. 当 check_duplicate=False 且 update_on_duplicate=True 时，由于 check_duplicate=False，直接插入全部data。
-        3. 当 check_duplicate=True 且 duplicate_columns=[] 且 update_on_duplicate=True 时，获取数据库所有列（但排除`id`和`更新时间`列），按这些列（不含`id`和`更新时间`）排重插入，遇到重复数据时更新旧数据。
-        4. 当 check_duplicate=True 且 duplicate_columns=[] 且 update_on_duplicate=False 时，获取数据库所有列（但排除`id`和`更新时间`列），按这些列（不含`id`和`更新时间`）排重插入，不考虑是否更新旧数据。
-        5. 当 check_duplicate=True 且 duplicate_columns 指定了排重列且 update_on_duplicate=True 时，按 duplicate_columns 指定的列（但排除`id`和`更新时间`）排重插入，遇到重复数据时更新旧数据。
-        6. 当 check_duplicate=True 且 duplicate_columns 指定了排重列且 update_on_duplicate=False 时，按 duplicate_columns 指定的列（但排除`id`和`更新时间`）排重插入，不考虑是否更新旧数据。
-
+        准备插入SQL语句, 增加StatementCache缓存
         """
+        cache_key = (db_name, table_name, tuple(sorted(set_typ.items())), check_duplicate, tuple(duplicate_columns) if duplicate_columns else (), update_on_duplicate)
+        cached = self._prepared_statements.get(cache_key)
+        if cached:
+            return cached
         # 获取所有列名（排除id）
         all_columns = [col for col in set_typ.keys()
                        if col.lower() != 'id']
-
-        # 情况1-2：不检查重复
         if not check_duplicate:
-            return self._build_simple_insert_sql(db_name, table_name, all_columns,
+            sql = self._build_simple_insert_sql(db_name, table_name, all_columns,
                                                  update_on_duplicate)
-
-        # 确定排重列（排除id和更新时间列）
-        dup_cols = duplicate_columns if duplicate_columns else [
-            col for col in all_columns
-            if col.lower() not in self.base_excute_col
-        ]
-
-        # 情况3-6：检查重复
-        return self._build_duplicate_check_sql(db_name, table_name, all_columns,
+        else:
+            dup_cols = duplicate_columns if duplicate_columns else [
+                col for col in all_columns
+                if col.lower() not in self.base_excute_col
+            ]
+            sql = self._build_duplicate_check_sql(db_name, table_name, all_columns,
                                                dup_cols, update_on_duplicate, set_typ)
+        self._prepared_statements[cache_key] = sql
+        return sql
 
     def _execute_batch_insert(
             self,
@@ -1220,10 +1214,8 @@ class MySQLUploader:
             transaction_mode: str,
             update_on_duplicate: bool = False
     ) -> Tuple[int, int, int]:
-        """执行批量插入操作"""
-
+        """执行批量插入操作，优化batch和hybrid模式"""
         def get_optimal_batch_size(total_rows: int) -> int:
-            # 根据数据量调整批量大小
             if total_rows <= 100:
                 return total_rows
             elif total_rows <= 1000:
@@ -1232,205 +1224,64 @@ class MySQLUploader:
                 return 1000
             else:
                 return 2000
-        
         batch_size = get_optimal_batch_size(len(data))
-        
-        # 获取所有列名（排除id列）
-        all_columns = [col for col in set_typ.keys()
-                       if col.lower() != 'id']
-
+        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
         total_inserted = 0
         total_skipped = 0
         total_failed = 0
-
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
-                for i in range(0, len(data), batch_size):
-                    batch = data[i:i + batch_size]
-                    batch_inserted, batch_skipped, batch_failed = self._process_batch(
-                        conn, cursor, db_name, table_name, batch, all_columns,
-                        sql, check_duplicate, duplicate_columns, batch_id,
-                        transaction_mode, update_on_duplicate
-                    )
-
-                    # 更新总统计
-                    total_inserted += batch_inserted
-                    total_skipped += batch_skipped
-                    total_failed += batch_failed
-
+                if transaction_mode == 'batch':
+                    for i in range(0, len(data), batch_size):
+                        batch = data[i:i + batch_size]
+                        values_list = []
+                        for row in batch:
+                            values = [row.get(col) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
+                                values += [row.get(col) for col in dup_cols]
+                            values_list.append(values)
+                        try:
+                            cursor.executemany(sql, values_list)
+                            conn.commit()
+                            total_inserted += len(batch)
+                        except Exception as e:
+                            conn.rollback()
+                            total_failed += len(batch)
+                            logger.error('批量插入失败', {'库': db_name, '表': table_name, '错误': str(e)})
+                elif transaction_mode == 'hybrid':
+                    hybrid_n = 100  # 可配置
+                    for i in range(0, len(data), hybrid_n):
+                        batch = data[i:i + hybrid_n]
+                        for row in batch:
+                            try:
+                                values = [row.get(col) for col in all_columns]
+                                if check_duplicate and not update_on_duplicate:
+                                    dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
+                                    values += [row.get(col) for col in dup_cols]
+                                cursor.execute(sql, values)
+                                total_inserted += 1
+                            except Exception as e:
+                                conn.rollback()
+                                total_failed += 1
+                                logger.error('hybrid单行插入失败', {'库': db_name, '表': table_name, '错误': str(e)})
+                        conn.commit()
+                else:  # row模式
+                    for row in data:
+                        try:
+                            values = [row.get(col) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
+                                values += [row.get(col) for col in dup_cols]
+                            cursor.execute(sql, values)
+                            conn.commit()
+                            total_inserted += 1
+                        except Exception as e:
+                            conn.rollback()
+                            total_failed += 1
+                            logger.error('单行插入失败', {'库': db_name, '表': table_name, '错误': str(e)})
         return total_inserted, total_skipped, total_failed
 
-    def _process_batch(
-            self,
-            conn,
-            cursor,
-            db_name: str,
-            table_name: str,
-            batch: List[Dict],
-            all_columns: List[str],
-            sql: str,
-            check_duplicate: bool,
-            duplicate_columns: Optional[List[str]],
-            batch_id: Optional[str],
-            transaction_mode: str,
-            update_on_duplicate: bool = False
-    ) -> Tuple[int, int, int]:
-        """
-        处理单个批次的数据插入
-
-        :param conn: 数据库连接对象
-        :param cursor: 数据库游标对象
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param batch: 当前批次的数据（字典列表）
-        :param all_columns: 需要插入的所有列名
-        :param sql: 执行的SQL语句
-        :param check_duplicate: 是否检查重复
-        :param duplicate_columns: 排重列
-        :param batch_id: 批次ID
-        :param transaction_mode: 事务模式
-        :param update_on_duplicate: 遇到重复时是否更新
-        :return: (插入数, 跳过数, 失败数)
-        """
-        batch_inserted = 0
-        batch_skipped = 0
-        batch_failed = 0
-        batch_size = len(batch)
-        logger.debug('批次插入开始', {
-            '库': db_name,
-            '表': table_name,
-            '批次ID': batch_id,
-            '批次大小': batch_size,
-            '事务模式': transaction_mode,
-            'SQL预览': sql[:200],
-            '排重': check_duplicate,
-            '排重列': duplicate_columns,
-            '允许更新': update_on_duplicate,
-            '数据样例': self._shorten_for_log(batch, 2)
-        })
-        if transaction_mode == 'batch':
-            try:
-                for row_idx, row in enumerate(batch, 1):
-                    result = self._process_single_row(
-                        db_name, table_name, cursor, row, all_columns, sql,
-                        check_duplicate, duplicate_columns, update_on_duplicate
-                    )
-                    if result == 'inserted':
-                        batch_inserted += 1
-                    elif result == 'skipped':
-                        batch_skipped += 1
-                    else:
-                        batch_failed += 1
-                conn.commit()
-                logger.debug('批次插入成功', {
-                    '库': db_name,
-                    '表': table_name,
-                    '批次ID': batch_id,
-                    '插入': batch_inserted,
-                    '跳过': batch_skipped,
-                    '失败': batch_failed
-                })
-            except Exception as e:
-                conn.rollback()
-                batch_failed += len(batch)
-                logger.error('批次插入失败', {
-                    '库': db_name,
-                    '表': table_name,
-                    '批次ID': batch_id,
-                    '错误': str(e),
-                    'SQL预览': sql[:200],
-                    '数据样例': self._shorten_for_log(batch, 2)
-                })
-        else:  # row 或 hybrid 模式
-            for row_idx, row in enumerate(batch, 1):
-                try:
-                    result = self._process_single_row(
-                        db_name, table_name, cursor, row, all_columns, sql,
-                        check_duplicate, duplicate_columns, update_on_duplicate
-                    )
-                    if result == 'inserted':
-                        batch_inserted += 1
-                    elif result == 'skipped':
-                        batch_skipped += 1
-                    else:
-                        batch_failed += 1
-                    conn.commit()
-                    logger.debug('单行插入成功', {
-                        '库': db_name,
-                        '表': table_name,
-                        '批次ID': batch_id,
-                        '行号': row_idx,
-                        '插入状态': result
-                    })
-                except Exception as e:
-                    conn.rollback()
-                    batch_failed += 1
-                    logger.error('单行插入失败', {
-                        '库': db_name,
-                        '表': table_name,
-                        '批次ID': batch_id,
-                        '行号': row_idx,
-                        '错误': str(e),
-                        'SQL预览': sql[:200],
-                        '数据': self._shorten_for_log(row)
-                    })
-        logger.debug('批次插入结束', {
-            '库': db_name,
-            '表': table_name,
-            '批次ID': batch_id,
-            '插入': batch_inserted,
-            '跳过': batch_skipped,
-            '失败': batch_failed,
-            '数据样例': self._shorten_for_log(batch, 2)
-        })
-        return batch_inserted, batch_skipped, batch_failed
-
-    def _process_single_row(
-            self,
-            db_name: str,
-            table_name: str,
-            cursor,
-            row: Dict,
-            all_columns: List[str],
-            sql: str,
-            check_duplicate: bool,
-            duplicate_columns: Optional[List[str]],
-            update_on_duplicate: bool = False
-    ) -> str:
-        """
-        处理单行数据插入
-
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param cursor: 数据库游标对象
-        :param row: 单行数据（字典）
-        :param all_columns: 需要插入的所有列名
-        :param sql: 执行的SQL语句
-        :param check_duplicate: 是否检查重复
-        :param duplicate_columns: 排重列
-        :param update_on_duplicate: 遇到重复时是否更新
-        :return: 'inserted' | 'skipped' | 'failed'
-        """
-        try:
-            # 构造参数
-            values = [row.get(col) for col in all_columns]
-            if check_duplicate:
-                # 需要为 WHERE NOT EXISTS 语句补充参数
-                if not update_on_duplicate:
-                    # duplicate_columns 为空时，默认用所有列（排除id/更新时间）
-                    dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
-                    values = values + [row.get(col) for col in dup_cols]
-            cursor.execute(sql, values)
-        except Exception as e:
-            logger.error('单行插入失败', {
-                '库': db_name,
-                '表': table_name,
-                'row': self._shorten_for_log(row),
-                '错误': str(e)
-            })
-            return 'failed'
-        return 'inserted'
-
     def close(self) -> None:
         """
         关闭连接池并清理资源
@@ -1441,10 +1292,10 @@ class MySQLUploader:
         try:
             if hasattr(self, 'pool') and self.pool is not None:
                 try:
-                    self.pool.close()
+                    # self.pool.close()  # PooledDB 没有 close 方法
+                    self.pool = None
                 except Exception as e:
                     logger.warning('关闭连接池时出错', {'error': str(e)})
-                self.pool = None
                 logger.info('连接池关闭', {'uploader.py': '连接池关闭'})
         except Exception as e:
             logger.error('关闭连接池失败', {'error': str(e)})
@@ -1452,11 +1303,12 @@ class MySQLUploader:
 
     def _check_pool_health(self) -> bool:
         """
-        检查连接池健康状态
-        :return: 连接池健康返回True，否则返回False
+        检查连接池健康状态，防止连接泄露
         """
         conn = None
         try:
+            if not hasattr(self, 'pool') or self.pool is None:
+                return False
             conn = self.pool.connection()
             conn.ping(reconnect=True)
             logger.debug('连接池健康检查通过')
@@ -1465,12 +1317,13 @@ class MySQLUploader:
             logger.warning('连接池健康检查失败', {'error': str(e)})
             return False
         finally:
-            if conn:
+            if conn is not None:
                 try:
                     conn.close()
                 except Exception as e:
                     logger.warning('关闭连接时出错', {'error': str(e)})
 
+    @staticmethod
     def retry_on_failure(max_retries: int = 3, delay: int = 1):
         """
         通用重试装饰器

mdbq/spider/aikucun.py

@@ -3,10 +3,8 @@ import datetime
 import requests
 import json
 import os
-import sys
 import re
 import time
-import warnings
 import platform
 import getpass
 from selenium import webdriver
@@ -15,20 +13,18 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
 import pymysql
-import pandas as pd
-from mdbq.log import spider_logging
-from mdbq.mysql import mysql
+from mdbq.mysql import uploader
 from mdbq.mysql import s_query
 from mdbq.config import config
 from mdbq.other import ua_sj
 from mdbq.other import otk
+from mdbq.log import mylogger
 
 dir_path = os.path.expanduser("~")
 config_file = os.path.join(dir_path, 'spd.txt')
 content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
-# m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 uld = uploader.MySQLUploader(username=username, password=password, host=host, port=int(port), pool_size=10)
 # 实例化一个数据查询类，用来获取 cookies 表数据
 download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
@@ -188,11 +184,6 @@ class AikuCun:
             '更新时间': 'timestamp'
         }
         # 更新至数据库记录
-        # m_engine.dict_to_mysql(
-        #     db_name=self.db_name,
-        #     table_name=self.table_name,
-        #     dict_data=self.token,
-        # )
         uld.upload_data(
             db_name=self.db_name,
             table_name=self.table_name,
@@ -429,15 +420,6 @@ class AikuCun:
             drop_dup = ['日期', '平台', '店铺名称', '商品款号', '访客量']
         else:
             drop_dup = ['日期', '平台', '店铺名称', '条码']
-        # m_engine.insert_many_dict(
-        #     db_name=db_name,
-        #     table_name=table_name,
-        #     dict_data_list=_results,
-        #     icm_update=drop_dup,  # 唯一组合键
-        #     # unique_main_key=['人群id'],
-        #     set_typ=set_typ,
-        #     allow_not_null=False,  # 创建允许插入空值的列
-        # )
         uld.upload_data(
             db_name=db_name,
             table_name=table_name,

{mdbq-3.11.1.dist-info → mdbq-3.11.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.11.1
+Version: 3.11.3
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.11.1.dist-info → mdbq-3.11.3.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=yfllhgz9Co6QQbMUq9eHkXYCazHcxgzu3Z4cqXTNmwo,18
+mdbq/__version__.py,sha256=SerN98H6Mx8rHVh-jf2Nmc7iZHb02NHGVphB1O5jKwE,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=fdotW8qdAyDB13p7r3p6AGBkavcHnf6hIvSMtcS7vqE,179875
+mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=07sstIeaIQUJXwpMwmxppRI7kW7QwZFnv4Rr3UDlyUs,24133
+mdbq/log/mylogger.py,sha256=HuxLBCXjm6fZrxYE0rdpUCz359WGeqOX0vvg9jTuRY4,24126
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=ibmxpzenhPgT_ei61TjQB2ZxYs9ztkG_ygbLSa8RIlM,32990
-mdbq/mysql/mysql.py,sha256=Lfy9PsEdgmdRtcG_UUgegH3bFTJPhByTWkcAYl8G6m0,56788
+mdbq/mysql/deduplicator.py,sha256=bIV010UkFfSUONY6-756x3tDVO4k6q3pqxoY3Z2xT-k,32990
+mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
-mdbq/mysql/uploader.py,sha256=3Ci34yVlLd7odqHLqnBRnkVMKM2Po26LJvtCpN-lBA4,66489
+mdbq/mysql/uploader.py,sha256=3fXyNA0GzBNaadAh6cOgbuUEvY4IAhKn4apgbkToEno,61321
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -23,8 +23,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
-mdbq-3.11.1.dist-info/METADATA,sha256=Nf8bnoO4TD0fo7xI6sVypTo2xHGy3ldHAQqfooRq_40,364
-mdbq-3.11.1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.11.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.11.1.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
+mdbq-3.11.3.dist-info/METADATA,sha256=tgDHEyJKxO0ML-gUTBap1b6yP-xv5sEA_SsfVJ_31C0,364
+mdbq-3.11.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.11.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.11.3.dist-info/RECORD,,