mdbq 4.0.42__py3-none-any.whl → 4.0.44__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.42'
+VERSION = '4.0.44'

mdbq/aggregation/query_data.py

@@ -972,6 +972,7 @@ class MysqlDatasQuery:
             '用户年龄': 'varchar(100)',
             '人群分类': 'varchar(100)',
         }
+        df.fillna(0, inplace=True)
         return df, {
             'db_name': db_name,
             'table_name': table_name,
@@ -1066,7 +1067,7 @@ class MysqlDatasQuery:
             )
         df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
         df['是否品牌词'] = df['词名字_词包名字'].str.contains('万里马|wanlima', regex=True)
-        df['是否品牌词'] = df['是否品牌词'].apply(lambda x: '品牌词' if x else '')
+        df['是否品牌词'] = df['是否品牌词'].apply(lambda x: '品牌词' if x else '-')
         dir_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\分类配置文件.xlsx'
         dir_file2 = '/Volumes/时尚事业部/01.运营部/0-电商周报-每周五更新/分类配置文件.xlsx'
         if not os.path.isfile(dir_file):
@@ -1514,8 +1515,6 @@ class MysqlDatasQuery:
             projection=projection,
         )
         df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-        df = df[df['日期'] == pd.to_datetime('2024-12-12')]
-
         df_set['商品id'] = df_set['商品id'].astype('int64')
         df['商品id'] = df['商品id'].astype('int64')
         df_set.sort_values('商品id', ascending=False, ignore_index=True, inplace=True)
@@ -1888,11 +1887,15 @@ class MysqlDatasQuery:
             end_date=end_date,
             projection=projection,
         )
-        df.pop('spu_id')  # 删除推广表的 spu id
+        if 'spu_id' in df.columns:
+            df = df.drop(columns=['spu_id'])  # 删除原有 spu_id，避免冲突
         df = pd.merge(df, df_sku, how='left', left_on='跟单sku_id', right_on='sku_id')
-        df.pop('sku_id')  # 删除聚合后合并进来的 sku id，实际使用 跟单sku_id
-        p = df.pop('spu_id')
-        df.insert(loc=3, column='spu_id', value=p)
+        df = df.drop(columns=['sku_id'])  # 删除 merge 进来的 sku_id
+        df['spu_id'] = df['spu_id'].fillna(0)  # 填充 spu_id 空值
+        # 调整 spu_id 到第3列
+        cols = list(df.columns)
+        cols.insert(3, cols.pop(cols.index('spu_id')))
+        df = df[cols]
         set_typ = {
             '日期': 'date',
             '店铺名称': 'varchar(100)',
@@ -2049,9 +2052,9 @@ class MysqlDatasQuery:
         )
         df = pd.merge(df, df_lin, how='left', left_on='计划id', right_on='计划id')
         df['k_是否品牌词'] = df['关键词'].str.contains('万里马|wanlima', regex=True)
-        df['k_是否品牌词'] = df['k_是否品牌词'].apply(lambda x: '品牌词' if x else '')
+        df['k_是否品牌词'] = df['k_是否品牌词'].apply(lambda x: '品牌词' if x else '-')
         df['s_是否品牌词'] = df['搜索词'].str.contains('万里马|wanlima', regex=True)
-        df['s_是否品牌词'] = df['s_是否品牌词'].apply(lambda x: '品牌词' if x else '')
+        df['s_是否品牌词'] = df['s_是否品牌词'].apply(lambda x: '品牌词' if x else '-')
         set_typ = {
             '日期': 'date',
             '产品线': 'varchar(100)',
@@ -3737,6 +3740,7 @@ def query3(months=1, download_manager=None):
     sdq = MysqlDatasQuery(download_manager=download_manager)  # 实例化数据处理类
     sdq.months = months  # 设置数据周期， 1 表示近 2 个月
     sdq.spph(db_name='聚合数据', table_name='天猫_商品排行')
+    sdq.item_up(db_name='聚合数据', table_name='淘宝店铺货品')
 
 
 def main(months=3):
@@ -3774,4 +3778,4 @@ if __name__ == '__main__':
         )
     sdq = MysqlDatasQuery(download_manager=download_manager) 
     sdq.months = 3
-    sdq.global_insights(db_name='聚合数据', table_name='全域洞察')
+    sdq.item_up(db_name='聚合数据', table_name='淘宝店铺货品')

mdbq/mysql/uploader.py

@@ -553,18 +553,112 @@ class MySQLUploader:
             logger.error('无效的日期格式', {'值': value})
         raise ValueError(f"无效的日期格式: `{value}`")
 
+    def _convert_to_int(self, value):
+        """
+        尝试将value转换为int
+        """
+        # 处理numpy/pandas标量
+        if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
+            try:
+                value = value.item()
+            except Exception:
+                pass
+        elif hasattr(value, 'value') and not isinstance(value, str):
+            try:
+                extracted_value = value.value
+                if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').isdigit():
+                    value = extracted_value
+            except Exception:
+                pass
+        try:
+            return int(value)
+        except (ValueError, TypeError):
+            try:
+                return int(float(value))
+            except (ValueError, TypeError):
+                raise
+
+    def _convert_to_float(self, value):
+        """
+        尝试将value转换为float，兼容常见数值类型。
+        """
+        if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
+            try:
+                value = value.item()
+            except Exception:
+                pass
+        elif hasattr(value, 'value') and not isinstance(value, str):
+            try:
+                extracted_value = value.value
+                if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').replace('e', '').replace('E', '').isdigit():
+                    value = extracted_value
+            except Exception:
+                pass
+        return float(value)
+
+    def _convert_to_decimal(self, value):
+        """
+        尝试将value转换为Decimal，兼容常见数值类型。
+        """
+        if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
+            try:
+                value = value.item()
+            except Exception:
+                pass
+        elif hasattr(value, 'value') and not isinstance(value, str):
+            try:
+                extracted_value = value.value
+                if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').replace('e', '').replace('E', '').isdigit():
+                    value = extracted_value
+            except Exception:
+                pass
+        return Decimal(str(value))
+
+    def _truncate_str(self, str_value, max_len):
+        """
+        截断字符串到指定字节长度（utf-8）。
+        """
+        return str_value.encode('utf-8')[:max_len].decode('utf-8', 'ignore')
+
     def _validate_value(self, value: Any, column_type: str, allow_null: bool, db_name: str = None, table_name: str = None, col_name: str = None) -> Any:
         """
         根据列类型验证并转换数据值
         """
         column_type_lower = column_type.lower() if column_type else ''
         # 统一判断None/NaN
+        if value == '':
+            if any(t in column_type_lower for t in ['varchar', 'text', 'char', 'mediumtext', 'longtext']):
+                return ""
         if value == '' or pd.isna(value) or (isinstance(value, (float, Decimal)) and math.isinf(value)):
-            if allow_null:
-                return None
-            else:
-                logger.error("该列不允许为空值", {"库": db_name, "表": table_name, "列": col_name, "值": value})
+            # 兜底填充值映射
+            fallback_map = {
+                'int': 0,
+                'bigint': 0,
+                'tinyint': 0,
+                'smallint': 0,
+                'mediumint': 0,
+                'decimal': 0.0,
+                'float': 0.0,
+                'double': 0.0,
+                'date': '1970-01-01',
+                'datetime': '1970-01-01 00:00:00',
+                'timestamp': '1970-01-01 00:00:00',
+                'json': '{}',
+                'varchar': 'none',
+                'text': 'none',
+                'char': 'none',
+                'mediumtext': 'none',
+                'longtext': 'none',
+            }
+            fallback = 'none'
+            for typ, val in fallback_map.items():
+                if typ in column_type_lower:
+                    fallback = val
+                    break
+            if not allow_null:
+                logger.warning("该列不允许为空值", {"库": db_name, "表": table_name, "allow_null": allow_null, "列": col_name, "值": value, "兜底值": fallback})
                 raise ValueError("该列不允许为空值")
+            return fallback
 
         original_value = value
 
@@ -576,25 +670,7 @@ class MySQLUploader:
         # 数值类型验证
         elif 'int' in column_type_lower:
             try:
-                # 安全地处理各种数值类型
-                if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
-                    # numpy 标量类型
-                    try:
-                        value = value.item()
-                    except (ValueError, TypeError):
-                        # 如果不是标量，保持原值
-                        pass
-                elif hasattr(value, 'value') and not isinstance(value, str):
-                    # pandas 或其他有 value 属性的对象
-                    try:
-                        extracted_value = value.value
-                        # 验证提取的值是数值类型
-                        if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').isdigit():
-                            value = extracted_value
-                    except (ValueError, TypeError, AttributeError):
-                        # 如果提取失败，保持原值
-                        pass
-                return int(value)
+                return self._convert_to_int(value)
             except (ValueError, TypeError):
                 logger.error(f"值 `{value}` 无法转换为整数", {"库": db_name, "表": table_name, "列": col_name})
                 raise ValueError(f"值 `{value}` 无法转换为整数")
@@ -602,64 +678,23 @@ class MySQLUploader:
             # 百分比字符串处理
             if isinstance(value, str) and '%' in value:
                 try:
-                    # 仅当值是'xx.xx%'格式时才转换
                     if re.match(r'^-?\d+(\.\d+)?%$', value.strip()):
                         value = float(value.strip().replace('%', '')) / 100
                     else:
-                        # 不符合格式的百分比字符串，保留原始值
                         logger.warning("百分比字符串不符合格式，跳过转换", {"库": db_name, "表": table_name, "列": col_name, "原始": original_value})
                         value = original_value
                 except (ValueError, TypeError):
                     logger.warning("百分比字符串转换失败，保留原始值", {"库": db_name, "表": table_name, "列": col_name, "原始": original_value})
                     value = original_value
-
             try:
-                # 检查是否为Decimal类型
                 if 'decimal' in column_type_lower:
                     precision, scale = self._get_decimal_scale(column_type)
-                    # 安全地处理各种数值类型
-                    if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
-                        # numpy 标量类型
-                        try:
-                            value = value.item()
-                        except (ValueError, TypeError):
-                            # 如果不是标量，保持原值
-                            pass
-                    elif hasattr(value, 'value') and not isinstance(value, str):
-                        # pandas 或其他有 value 属性的对象
-                        try:
-                            extracted_value = value.value
-                            # 验证提取的值是数值类型
-                            if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').replace('e', '').replace('E', '').isdigit():
-                                value = extracted_value
-                        except (ValueError, TypeError, AttributeError):
-                            # 如果提取失败，保持原值
-                            pass
-                    value_decimal = Decimal(str(value))
-                    # 检查整数部分长度
+                    value_decimal = self._convert_to_decimal(value)
                     if len(value_decimal.as_tuple().digits) - abs(value_decimal.as_tuple().exponent) > precision - scale:
                         raise ValueError(f"整数部分超出范围")
                     return value_decimal
-                else: # float/double
-                    # 安全地处理各种数值类型
-                    if hasattr(value, 'item') and callable(getattr(value, 'item', None)):
-                        # numpy 标量类型
-                        try:
-                            value = value.item()
-                        except (ValueError, TypeError):
-                            # 如果不是标量，保持原值
-                            pass
-                    elif hasattr(value, 'value') and not isinstance(value, str):
-                        # pandas 或其他有 value 属性的对象
-                        try:
-                            extracted_value = value.value
-                            # 验证提取的值是数值类型
-                            if isinstance(extracted_value, (int, float, str)) and str(extracted_value).replace('.', '').replace('-', '').replace('e', '').replace('E', '').isdigit():
-                                value = extracted_value
-                        except (ValueError, TypeError, AttributeError):
-                            # 如果提取失败，保持原值
-                            pass
-                    return float(value)
+                else:  # float/double
+                    return self._convert_to_float(value)
             except (ValueError, TypeError, InvalidOperation) as e:
                 logger.error(f"值 `{value}` 无法转换为数值类型: {e}", {"库": db_name, "表": table_name, "列": col_name})
                 raise ValueError(f"值 `{value}` 无法转换为数值类型: {e}")
@@ -670,12 +705,10 @@ class MySQLUploader:
                 max_len = int(re.search(r'\((\d+)\)', column_type).group(1))
                 if len(str_value.encode('utf-8')) > max_len:
                     logger.warning(f"列`{col_name}`的值`{str_value}`长度({len(str_value.encode('utf-8'))})超出varchar({max_len})限制，将进行截断", {"库": db_name, "表": table_name})
-                    return str_value.encode('utf-8')[:max_len].decode('utf-8', 'ignore')
+                    return self._truncate_str(str_value, max_len)
             except (AttributeError, IndexError):
-                # 没有找到长度定义，不截断
                 pass
             return str_value
-        
         return value
 
     @_execute_with_retry

{mdbq-4.0.42.dist-info → mdbq-4.0.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.42
+Version: 4.0.44
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.42.dist-info → mdbq-4.0.44.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=oNzbjnM_gPKVuYbTskX3n8fcwlgDsOEOgDiMJaHcsIU,18
+mdbq/__version__.py,sha256=Yq_JgKwKONwVexEyE66trDbripXgbesAkvt1eQ___20,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=WtTFMN78jn43Y-nBTPAXhAK56w3wDuv_cj4YtzzGbZk,169797
+mdbq/aggregation/query_data.py,sha256=ZWLJghNiEtyA4rvgUqMCLorY0R4-Likd6i4mVMuOni0,170025
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=kPe3wsQNaB1slfX-Z7VMqzZoMoqPfc7ylYXZDBeFzzI,24945
 mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -11,7 +11,7 @@ mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=1wJ3HVjHEF6FA-bVeeesRlsf73CZSvVTEQ51CF1OsE4,46786
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=d2ocnCTyBkAJ4LjFLOatb0VgkJ28g70fPXJropRlH1s,86370
+mdbq/mysql/uploader.py,sha256=LPfYEj7ywoAynY2Nl9gg0IurgIWd_bTwkda2ifD1TeE,86443
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=XptHjGzbout9IYzWAOQUpMMV5qEgLTU8pL1ZGt8oNEA,21868
-mdbq-4.0.42.dist-info/METADATA,sha256=XEoFYTnRfRqAv-rIatsbqCFRTXAP9vP0Zv6g-KzIOWU,364
-mdbq-4.0.42.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.42.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.42.dist-info/RECORD,,
+mdbq-4.0.44.dist-info/METADATA,sha256=6BCrO5mef08KVeODcA3rgk-gOsNtI_8_CTh0mghCKeE,364
+mdbq-4.0.44.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.44.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.44.dist-info/RECORD,,