mdbq 4.1.6__py3-none-any.whl → 4.1.8__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.1.6'
+VERSION = '4.1.8'

mdbq/mysql/uploader.py

@@ -784,11 +784,18 @@ class MySQLUploader:
                 'char': 'none',
                 'mediumtext': 'none',
                 'longtext': 'none',
+                'enum': None,  # enum类型需要特殊处理，使用第一个可选值
+                'set': '',     # set类型默认为空字符串
             }
             fallback = 'none'
             for typ, val in fallback_map.items():
                 if typ in column_type_lower:
-                    fallback = val
+                    if typ == 'enum' and val is None:
+                        # 对于enum类型，使用第一个可选值作为默认值
+                        enum_values = re.findall(r"['\"]([^'\"]*)['\"]", column_type)
+                        fallback = enum_values[0] if enum_values else 'none'
+                    else:
+                        fallback = val
                     break
             if not allow_null:
                 logger.warning("该列不允许为空值", {"库": db_name, "表": table_name, "allow_null": allow_null, "列": col_name, "值": value, "兜底值": fallback})
@@ -833,6 +840,33 @@ class MySQLUploader:
             except (ValueError, TypeError, InvalidOperation) as e:
                 logger.error(f"值 `{value}` 无法转换为数值类型: {e}", {"库": db_name, "表": table_name, "列": col_name})
                 raise ValueError(f"值 `{value}` 无法转换为数值类型: {e}")
+        # ENUM类型验证
+        elif 'enum' in column_type_lower:
+            # 提取enum的可选值，支持单引号和双引号
+            enum_values = re.findall(r"['\"]([^'\"]*)['\"]", column_type)
+            str_value = str(value).strip()
+            if str_value not in enum_values:
+                logger.error(f"值 `{str_value}` 不在enum允许的值中: {enum_values}", 
+                            {"库": db_name, "表": table_name, "列": col_name, "列类型": column_type})
+                raise ValueError(f"值 `{str_value}` 不在enum允许的值中: {enum_values}")
+            return str_value
+        # SET类型验证
+        elif 'set' in column_type_lower:
+            # 提取set的可选值，支持单引号和双引号
+            set_values = re.findall(r"['\"]([^'\"]*)['\"]", column_type)
+            str_value = str(value).strip()
+            # SET类型可以是多个值的组合，用逗号分隔
+            if ',' in str_value:
+                input_values = [v.strip() for v in str_value.split(',')]
+            else:
+                input_values = [str_value]
+            
+            for val in input_values:
+                if val and val not in set_values:
+                    logger.error(f"值 `{val}` 不在set允许的值中: {set_values}", 
+                                {"库": db_name, "表": table_name, "列": col_name, "列类型": column_type})
+                    raise ValueError(f"值 `{val}` 不在set允许的值中: {set_values}")
+            return str_value
         # 字符串类型验证
         elif 'varchar' in column_type_lower:
             str_value = str(value)
@@ -1134,21 +1168,27 @@ class MySQLUploader:
     ) -> Tuple[List[Dict], Dict[str, str]]:
         """
         准备要上传的数据，验证并转换数据类型
+        根据set_typ自动处理所有数据类型的列：补齐缺失的列并丢弃多余的列
         """
+        # set_typ的键清洗
+        if not set_typ:
+            set_typ = {}
+        normalized_set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
+        
         # 统一数据格式为字典列表
         if isinstance(data, pd.DataFrame):
             try:
                 if self.case_sensitive:
-                    data.columns = [col for col in data.columns]
+                    data.columns = [self._validate_identifier(col) for col in data.columns]
                 else:
-                    data.columns = [col.lower() for col in data.columns]
+                    data.columns = [self._validate_identifier(col).lower() for col in data.columns]
                 data = data.replace({pd.NA: None}).to_dict('records')
             except Exception as e:
-                logger.error('数据转字典时发生错误', {
+                logger.error('DataFrame处理时发生错误', {
                     'error': str(e),
                     'data': self._shorten_for_log(data),
                 })
-                raise ValueError(f"数据转字典时发生错误: {e}")
+                raise ValueError(f"DataFrame处理时发生错误: {e}")
         elif isinstance(data, dict):
             if self.case_sensitive:
                 data = [{k: v for k, v in data.items()}]
@@ -1168,34 +1208,65 @@ class MySQLUploader:
         # 统一处理原始数据中列名的特殊字符
         data = self.normalize_column_names(data)
 
-        if not set_typ:
+        if not normalized_set_typ:
             logger.warning('set_typ为空, 将自动推断数据类型, 可能存在数据类型识别错误')
-        # set_typ的键清洗
-        if not set_typ:
-            set_typ = {}
-        set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
 
-        # 新实现：严格按set_typ顺序过滤，后补充data中有但set_typ没有的列
+        # 根据set_typ处理所有数据的列：严格按set_typ定义的列进行过滤
         filtered_set_typ = {}
         data_columns = list(data[0].keys()) if data and len(data) > 0 else []
-        # 先按set_typ顺序
-        for col in set_typ:
-            if col in data_columns:
-                filtered_set_typ[col] = set_typ[col]
-        # 再补充data中有但set_typ没有的列
-        for col in data_columns:
-            if col not in filtered_set_typ:
-                # 推断类型
-                sample_values = [row[col] for row in data if col in row and row[col] is not None][:5]
-                inferred_type = None
-                for val in sample_values:
-                    inferred_type = self._infer_data_type(val, no_log=True)
-                    if inferred_type:
-                        break
-                if not inferred_type:
-                    inferred_type = 'VARCHAR(255)'
-                filtered_set_typ[col] = inferred_type
-                logger.debug(f"自动推断列 `{col}` 的数据类型为: `{inferred_type}`")
+        
+        if normalized_set_typ:
+            # 严格按照set_typ定义的列进行过滤，排除id列
+            for col in normalized_set_typ:
+                if (self.case_sensitive and col == 'id') or (not self.case_sensitive and col.lower() == 'id'):
+                    continue
+                filtered_set_typ[col] = normalized_set_typ[col]
+            
+            # 对所有数据行进行列处理：补齐缺失列，丢弃多余列
+            processed_data = []
+            for row in data:
+                processed_row = {}
+                # 只保留set_typ中定义的列
+                for col in filtered_set_typ:
+                    if col in row:
+                        processed_row[col] = row[col]
+                    else:
+                        processed_row[col] = None  # 缺失列用None填充
+                processed_data.append(processed_row)
+            data = processed_data
+            
+            # 检查是否有丢弃的列
+            dropped_columns = [col for col in data_columns if col not in filtered_set_typ]
+            if dropped_columns:
+                logger.warning('数据中存在set_typ未定义的列并已被丢弃', {
+                    '库': db_name,
+                    '表': table_name,
+                    '丢弃列': dropped_columns,
+                    # '保留列': list(filtered_set_typ.keys())
+                })
+            
+            logger.debug('数据列处理完成', {
+                '库': db_name,
+                '表': table_name,
+                '原始列': data_columns,
+                '目标列': list(filtered_set_typ.keys()),
+                '丢弃列': dropped_columns
+            })
+        else:
+            # 如果set_typ为空，则推断所有数据列的类型
+            for col in data_columns:
+                if col not in filtered_set_typ:
+                    # 推断类型
+                    sample_values = [row[col] for row in data if col in row and row[col] is not None][:5]
+                    inferred_type = None
+                    for val in sample_values:
+                        inferred_type = self._infer_data_type(val, no_log=True)
+                        if inferred_type:
+                            break
+                    if not inferred_type:
+                        inferred_type = 'VARCHAR(255)'
+                    filtered_set_typ[col] = inferred_type
+                    logger.debug(f"自动推断列 `{col}` 的数据类型为: `{inferred_type}`")
 
         prepared_data = []
         for row_idx, row in enumerate(data, 1):
@@ -1205,11 +1276,15 @@ class MySQLUploader:
                 if (self.case_sensitive and col_name == 'id') or (not self.case_sensitive and col_name.lower() == 'id'):
                     continue
                 if col_name not in row:
-                    if not allow_null:
-                        error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`"
-                        logger.error(error_msg, {'row': self._shorten_for_log(row)})
-                        raise ValueError(error_msg)
-                    prepared_row[col_name] = None
+                    # 对于缺失的列，使用None作为默认值，在_validate_value中会根据allow_null和列类型进行进一步处理
+                    try:
+                        prepared_row[col_name] = self._validate_value(None, filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
+                    except ValueError as e:
+                        if not allow_null:
+                            error_msg = f"行号:{row_idx} -> 缺失列: `{col_name}`, 且不允许空值"
+                            logger.error(error_msg, {'row': self._shorten_for_log(row)})
+                            raise ValueError(error_msg)
+                        prepared_row[col_name] = None
                 else:
                     try:
                         prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null, db_name, table_name, col_name)
@@ -2296,6 +2371,47 @@ class MySQLUploader:
         })
         return validated_keys
 
+    @staticmethod
+    def process_df_columns(
+            df: pd.DataFrame, 
+            columns: List[str], 
+            default_value: Any = 0
+    ) -> pd.DataFrame:
+        """
+        处理DataFrame的列，补齐缺失的列并丢弃多余的列
+        
+        :param df: 要处理的DataFrame
+        :param columns: 所需的列名列表，注意不处理大小写
+        :param default_value: 缺失列的填充值，默认为None
+        :return: 处理后的DataFrame
+        """
+        if df is None or not isinstance(df, pd.DataFrame) or not isinstance(columns, list) or not columns:
+            return df
+        
+        # 获取当前列名
+        current_columns = list(df.columns)
+        
+        # 找出需要添加的列和需要删除的列
+        missing_columns = [col for col in columns if col not in current_columns]
+        extra_columns = [col for col in current_columns if col not in columns]
+        
+        # 复制DataFrame
+        result_df = df.copy()
+        
+        # 删除多余的列
+        if extra_columns:
+            result_df = result_df.drop(columns=extra_columns)
+        
+        # 添加缺失的列
+        if missing_columns:
+            for col in missing_columns:
+                result_df[col] = default_value
+        
+        # 按照指定顺序重新排列列
+        result_df = result_df.reindex(columns=columns)
+        
+        return result_df
+
 
 def main():
     dir_path = os.path.expanduser("~")

{mdbq-4.1.6.dist-info → mdbq-4.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: mdbq
-Version: 4.1.6
+Version: 4.1.8
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.1.6.dist-info → mdbq-4.1.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=7c62s32-DiiicOsmr_3thHK6QH89lYKau8J0SY1GH-g,17
+mdbq/__version__.py,sha256=Jd5sP6XMAzuwfeUQJhdmEnXOsZWP5LcueloV0VDjdww,17
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
 mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -15,7 +15,7 @@ mdbq/mysql/deduplicator.py,sha256=2fugLyKs_xkvYvoG0C0hRYbJ_w8-4oa1FJ_vavoD7Qo,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=N2xHJf2CiUXjXIVBemdst-wamIP3908EGAJOFG13fCU,50475
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=FG_4btNwTjbCqZFeIigCfar7r-OOA7VkyuJsOOC9WLw,111539
+mdbq/mysql/uploader.py,sha256=VhI_VKpJ1fld6dzdDv_Q1gN56mrcPnDsXAdnDgFAqeg,117186
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.1.6.dist-info/METADATA,sha256=2dRtmGxtsWYYcheH8sXkFexnBq687lwKffpmQIdWtrU,363
-mdbq-4.1.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mdbq-4.1.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.1.6.dist-info/RECORD,,
+mdbq-4.1.8.dist-info/METADATA,sha256=rBRP7HOwmYzaA380L7fXLBlAFQn2xHgFYObmlMSLa2I,363
+mdbq-4.1.8.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.1.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.1.8.dist-info/RECORD,,

{mdbq-4.1.6.dist-info → mdbq-4.1.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any