mdbq 4.0.63__py3-none-any.whl → 4.0.65__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.63'
+VERSION = '4.0.65'

mdbq/mysql/s_query.py

@@ -779,6 +779,74 @@ class QueryDatas:
                 df[col] = df[col].astype(float)
         return df
 
+    def _convert_columns_to_lowercase(self, data: Union[pd.DataFrame, List[Dict[str, Any]]], 
+                                    lower_col: Optional[List[str]], 
+                                    return_format: str) -> Union[pd.DataFrame, List[Dict[str, Any]]]:
+        """
+        将指定列的值转换为小写。
+        
+        Args:
+            data: 原始数据，可以是DataFrame或列表字典
+            lower_col: 需要转换为小写的列名列表
+            return_format: 返回数据格式
+            
+        Returns:
+            转换后的数据
+        """
+        # 参数验证
+        if not lower_col or not data:
+            return data
+            
+        # 确保 lower_col 是列表类型
+        if not isinstance(lower_col, list):
+            logger.warning('lower_col 参数类型错误，应为列表', {'传入类型': type(lower_col).__name__})
+            return data
+            
+        try:
+            if return_format == 'df':
+                df = data.copy()
+                for col in lower_col:
+                    if not isinstance(col, str):
+                        logger.warning('列名必须是字符串类型', {'列名': col, '类型': type(col).__name__})
+                        continue
+                        
+                    if col in df.columns:
+                        # 只对字符串类型的列进行小写转换
+                        if df[col].dtype == 'object':
+                            # 更安全的空值处理：保持 None/NaN 不变，只转换非空字符串
+                            df[col] = df[col].apply(lambda x: str(x).lower() if pd.notna(x) and x is not None else x)
+                        else:
+                            logger.debug('列不是字符串类型，跳过小写转换', {'列名': col, '数据类型': df[col].dtype})
+                        logger.debug('列转换为小写', {'列名': col, '数据类型': df[col].dtype})
+                    else:
+                        logger.debug('指定的列不存在，跳过小写转换', {'列名': col, '可用列': list(df.columns)})
+                return df
+            else:  # list_dict格式
+                result = []
+                for row in data:
+                    new_row = row.copy()
+                    for col in lower_col:
+                        if not isinstance(col, str):
+                            logger.warning('列名必须是字符串类型', {'列名': col, '类型': type(col).__name__})
+                            continue
+                            
+                        if col in new_row and new_row[col] is not None:
+                            # 确保值是字符串类型再转换为小写
+                            try:
+                                new_row[col] = str(new_row[col]).lower()
+                            except Exception as e:
+                                logger.debug('值转换为小写失败', {'列名': col, '值': new_row[col], '错误': str(e)})
+                                # 保持原值不变
+                    result.append(new_row)
+                return result
+        except Exception as e:
+            logger.warning('小写转换失败', {
+                '错误类型': type(e).__name__,
+                '错误信息': str(e),
+                '指定列': lower_col
+            })
+            return data
+
     def __enter__(self):
         """上下文管理器入口"""
         return self
@@ -831,7 +899,8 @@ class QueryDatas:
             limit: Optional[int] = None,
             page_size: Optional[int] = None,
             date_column: Optional[str] = None,
-            return_format: Literal['df', 'list_dict'] = 'df'
+            return_format: Literal['df', 'list_dict'] = 'df',
+            lower_col: Optional[List[str]] = ['店铺名称']
             ) -> Union[pd.DataFrame, List[Dict[str, Any]]]:
         """
         从数据库表获取数据，支持列筛选、日期范围过滤和行数限制。
@@ -855,6 +924,7 @@ class QueryDatas:
             return_format: 返回数据格式
                          - 'df': 返回pandas DataFrame（默认）
                          - 'list_dict': 返回列表字典格式 [{列1:值, 列2:值, ...}, ...]
+            lower_col: 需要转换为小写的列名列表，默认['店铺名称']。如果为None或空列表，则不进行小写转换。
             
         Returns:
             根据return_format参数返回不同格式的数据：
@@ -874,6 +944,18 @@ class QueryDatas:
             logger.error('无效的return_format值', {'库': db_name, '表': table_name, '指定返回数据格式, 有效值应为: ': ', '.join(valid_formats)})
             return [] if return_format == 'list_dict' else pd.DataFrame()
 
+        # 验证lower_col参数
+        if lower_col is not None:
+            if not isinstance(lower_col, list):
+                logger.warning('lower_col 参数类型错误，应为列表，将使用默认值', {'传入类型': type(lower_col).__name__})
+                lower_col = ['店铺名称']
+            elif len(lower_col) > 0:
+                # 验证列表中的每个元素都是字符串
+                invalid_items = [item for item in lower_col if not isinstance(item, str)]
+                if invalid_items:
+                    logger.warning('lower_col 列表包含非字符串元素，将过滤掉', {'无效元素': invalid_items})
+                    lower_col = [item for item in lower_col if isinstance(item, str)]
+
         # 验证日期范围
         start_date, end_date = self._validate_date_range(start_date, end_date, db_name, table_name)
         
@@ -928,10 +1010,12 @@ class QueryDatas:
                         
                         if result:
                             if return_format == 'list_dict':
+                                result = self._convert_columns_to_lowercase(result, lower_col, return_format)
                                 return result
                             else:
                                 df = pd.DataFrame(result)
                                 df = self._convert_decimal_columns(df)
+                                df = self._convert_columns_to_lowercase(df, lower_col, return_format)
                                 return df
                         return [] if return_format == 'list_dict' else pd.DataFrame()
 
@@ -975,11 +1059,13 @@ class QueryDatas:
                     })
 
                     if return_format == 'list_dict':
+                        all_results = self._convert_columns_to_lowercase(all_results, lower_col, return_format)
                         return all_results
                     else:
                         df = pd.DataFrame(all_results)
                         if not df.empty:
                             df = self._convert_decimal_columns(df)
+                            df = self._convert_columns_to_lowercase(df, lower_col, return_format)
                         return df
 
         except Exception as e:

mdbq/mysql/uploader.py

@@ -267,7 +267,7 @@ class MySQLUploader:
         :return: 存在返回True，否则返回False
         :raises: 可能抛出数据库相关异常
         """
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         sql = 'SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s'
         conn = None
         try:
@@ -289,7 +289,7 @@ class MySQLUploader:
         :param db_name: 要创建的数据库名称
         :raises: 可能抛出数据库相关异常
         """
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         sql = f'CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}'
         conn = None
         try:
@@ -327,11 +327,12 @@ class MySQLUploader:
             logger.error('分表方式无效', {'表': table_name, '分表方式': partition_by})
             raise ValueError("分表方式必须是 'year' 或 'month' 或 'None'")
 
-    def _validate_identifier(self, identifier: str) -> str:
+    def _validate_identifier(self, identifier: str, is_database: bool = False) -> str:
         """
         验证并清理数据库标识符(表名、列名等)
 
         :param identifier: 要验证的标识符
+        :param is_database: 是否为数据库名，数据库名不能以数字开头
         :return: 清理后的安全标识符
         :raises ValueError: 当标识符无效时抛出
         """
@@ -349,9 +350,14 @@ class MySQLUploader:
             hash_suffix = hashlib.md5(identifier.encode('utf-8')).hexdigest()[:8]
             cleaned = f'unknown_col_{hash_suffix}'
         
-        # 确保标识符不以数字开头（MySQL要求）
-        if cleaned and cleaned[0].isdigit():
-            cleaned = f'col_{cleaned}'
+        # 数据库名不能以数字开头（MySQL要求），但表名和列名可以
+        if is_database and cleaned and cleaned[0].isdigit():
+            cleaned = f'db_{cleaned}'
+            logger.warning('为数字开头的数据库名添加db_前缀', {
+                '原始标识符': identifier,
+                '清理后': cleaned
+            })
+        
         mysql_keywords = {
             'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
             'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
@@ -379,7 +385,7 @@ class MySQLUploader:
             if time.time() - cached_time < self.metadata_cache_ttl:
                 logger.debug('表存在缓存命中', {'库': db_name, '表': table_name, '存在': result})
                 return result
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         table_name = self._validate_identifier(table_name)
         sql = """
                 SELECT TABLE_NAME 
@@ -413,7 +419,7 @@ class MySQLUploader:
         """
         创建数据表，优化索引创建方式
         """
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         table_name = self._validate_identifier(table_name)
         if not set_typ:
             logger.error('建表时未指定set_typ', {'库': db_name, '表': table_name})
@@ -859,7 +865,7 @@ class MySQLUploader:
         :return: 列名和数据类型字典 {列名: 数据类型}
         :raises: 可能抛出数据库相关异常
         """
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         table_name = self._validate_identifier(table_name)
         sql = """
         SELECT COLUMN_NAME, DATA_TYPE 
@@ -885,7 +891,7 @@ class MySQLUploader:
         """
         确保某列有索引，如果没有则创建。
         """
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         table_name = self._validate_identifier(table_name)
         column = self._validate_identifier(column)
         # 检查索引是否已存在
@@ -914,7 +920,7 @@ class MySQLUploader:
         获取表中所有UNIQUE KEY的列组合（不含主键）。
         返回：[[col1, col2], ...]
         """
-        db_name = self._validate_identifier(db_name)
+        db_name = self._validate_identifier(db_name, is_database=True)
         table_name = self._validate_identifier(table_name)
         sql = '''
             SELECT INDEX_NAME, COLUMN_NAME

{mdbq-4.0.63.dist-info → mdbq-4.0.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.63
+Version: 4.0.65
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.63.dist-info → mdbq-4.0.65.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=6Et4ZNNB7_Ld1K-A0R7SKzScGaPlzdt-ysSI2cj2u7E,18
+mdbq/__version__.py,sha256=1gP4SARV8VrJwdAE1ZRMMnoekF6Fe6v0VGYuInbT8h0,18
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=kPe3wsQNaB1slfX-Z7VMqzZoMoqPfc7ylYXZDBeFzzI,24945
 mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -7,9 +7,9 @@ mdbq/myconf/myconf.py,sha256=rHvQCnQRKhQ49AZBke-Z4v28hyOLmHt4MylIuB0H6yA,33516
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73074
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=rSlAIUNfTrI25vYjoq9OpEbzsbrzc0m8mxb_0G7h878,45496
+mdbq/mysql/s_query.py,sha256=hptv4YxnkqPruJkgNX8UxRRhisvSEnn8HzDGDLmCQFw,50231
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=sfgrHR3fKQ7ys-Y_cLCFBCFgwg3ZUL_fSJVu2suizOM,111658
+mdbq/mysql/uploader.py,sha256=68kJIrCnP2dJZ6ilb8MoFzuzGGWU_272WwPfaqt075A,112125
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -22,7 +22,7 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.0.63.dist-info/METADATA,sha256=FrAiUp_WT-eGT8lzHZxw7jnAjeemU9nDFrQFDVg95Gg,364
-mdbq-4.0.63.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.63.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.63.dist-info/RECORD,,
+mdbq-4.0.65.dist-info/METADATA,sha256=5tbI-X82C-RHIHkdxXt6thtOGZbDDhYmeil4N8uQWBQ,364
+mdbq-4.0.65.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.65.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.65.dist-info/RECORD,,