mdbq 2.6.1__py3-none-any.whl → 2.6.2__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -93,6 +93,18 @@ class MysqlDatasQuery:
         username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
         self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 
+    @staticmethod
+    def try_except(func):  # 在类内部定义一个异常处理方法
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                print(f'{func.__name__}, {e}')  # 将异常信息返回
+
+        return wrapper
+
+    @try_except
     def tg_wxt(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -118,6 +130,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def syj(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -142,6 +155,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def tg_rqbb(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -167,6 +181,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def tg_gjc(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -193,6 +208,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def tg_cjzb(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -220,6 +236,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def pxb_zh(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -245,6 +262,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def idbm(self):
         """ 用生意经日数据制作商品 id 和编码对照表 """
         data_values = self.download.columns_to_list(
@@ -255,6 +273,7 @@ class MysqlDatasQuery:
         df = pd.DataFrame(data=data_values)
         return df
 
+    @try_except
     def sp_picture(self):
         """ 用生意经日数据制作商品 id 和编码对照表 """
         data_values = self.download.columns_to_list(
@@ -265,6 +284,7 @@ class MysqlDatasQuery:
         df = pd.DataFrame(data=data_values)
         return df
 
+    @try_except
     def dplyd(self):
         """ 新旧版取的字段是一样的 """
         start_date, end_date = self.months_data(num=self.months)
@@ -288,6 +308,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def dplyd_old(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -310,6 +331,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def sp_cost(self):
         """ 电商定价 """
         data_values = self.download.columns_to_list(
@@ -320,6 +342,7 @@ class MysqlDatasQuery:
         df = pd.DataFrame(data=data_values)
         return df
 
+    @try_except
     def jdjzt(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -346,6 +369,8 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+
+    @try_except
     def jdqzyx(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -368,6 +393,8 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+
+    @try_except
     def jd_gjc(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -401,6 +428,8 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+
+    @try_except
     def sku_sales(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -422,6 +451,8 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+
+    @try_except
     def spu_sales(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -453,6 +484,7 @@ class MysqlDatasQuery:
         start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
         return pd.to_datetime(start_date), pd.to_datetime(end_date)
 
+    @try_except
     def tm_search(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -476,6 +508,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def zb_ccfx(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -520,6 +553,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def tg_by_day(self):
         """
         汇总各个店铺的推广数据，按日汇总
@@ -689,6 +723,7 @@ class MysqlDatasQuery:
         df = pd.concat(_datas, axis=0, ignore_index=True)
         return df
 
+    @try_except
     def aikucun_bd_spu(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -736,6 +771,7 @@ class MysqlDatasQuery:
         )
         return df
 
+    @try_except
     def dmp_crowd(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -1497,6 +1533,7 @@ class GroupBy:
             print(f'<{table_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
 
+    @try_except
     def ret_keyword(self, keyword, as_file=False):
         """ 推广关键词报表，关键词分类， """
         datas = [
@@ -1664,6 +1701,7 @@ class GroupBy:
                 break
         return result
 
+    @try_except
     def set_crowd(self, keyword, as_file=False):
         """ 推广人群报表，人群分类， """
         result_a = re.findall('_a$|_a_|_ai|^a_', str(keyword), re.IGNORECASE)
@@ -1699,6 +1737,7 @@ class GroupBy:
         if not is_res:
             return ''
 
+    @try_except
     def set_crowd2(self, keyword, as_file=False):
         """ 推广人群报表，人群分类， """
         datas = [
@@ -1796,7 +1835,7 @@ class GroupBy:
                 break
         return result
 
-    # @try_except
+    @try_except
     def performance(self, bb_tg=True):
          # print(self.data_tgyj)
         tg, syj, idbm, pic, cost = (
@@ -1832,6 +1871,8 @@ class GroupBy:
         df['毛利率'] = df.apply(lambda x: round((x['销售额'] - x['商品成本']) / x['销售额'], 4) if x['销售额'] > 0 else 0, axis=1)
         df['盈亏'] = df.apply(lambda x: x['商品毛利'] - x['花费'], axis=1)
         return df
+
+    @try_except
     def performance_concat(self, bb_tg=True):
         tg,  zb, pxb = self.data_tgyj['天猫汇总表调用'], self.data_tgyj['天猫_超级直播'], self.data_tgyj['天猫_品销宝账户报表']
         zb.rename(columns={
@@ -1880,6 +1921,7 @@ class GroupBy:
         )
         return df
 
+    @try_except
     def performance_jd(self, jd_tg=True):
         jdtg, sku_sales = self.data_jdtg['京东_京准通'], self.data_jdtg['京东_sku_商品明细']
         jdtg = jdtg.groupby(['日期', '跟单sku id'],
@@ -2285,6 +2327,6 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=24, is_juhe=False)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=0, is_juhe=False)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/clean/clean_upload.py

@@ -89,15 +89,30 @@ class DataClean:
             path = self.path
         report_names = [
             {
-                '文件简称': '商品排行',  # 文件名中包含的字符
-                '数据库名': '天猫_生意参谋3',
+                '文件简称': '商品排行_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
                 '集合名称': '商品排行',
             },
             {
-                '文件简称': '店铺来源_来源构成_万里马官方旗舰店',  # 文件名中包含的字符
-                '数据库名': '天猫_生意参谋3',
+                '文件简称': '店铺来源_来源构成_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
                 '集合名称': '店铺流量来源构成',
             },
+            {
+                '文件简称': '商品类目属性_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
+                '集合名称': '商品类目属性',
+            },
+            {
+                '文件简称': '商品主图视频_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
+                '集合名称': '商品主图视频',
+            },
+            {
+                '文件简称': '商品sku属性_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
+                '集合名称': '商品sku',
+            },
         ]
         for root, dirs, files in os.walk(path, topdown=False):
             for name in files:
@@ -126,7 +141,7 @@ class DataClean:
                         is_continue = True
                 if not is_continue:
                     continue
-                if name.endswith('.xls') and '商品排行_万里马官方旗舰店' in name:
+                if name.endswith('.xls') and '商品排行_' in name:
                     df = pd.read_excel(os.path.join(root, name), header=4)
                     if len(df) == 0:
                         print(f'{name} 报表数据为空')
@@ -135,15 +150,20 @@ class DataClean:
                     df.replace(to_replace=[','], value='', regex=True, inplace=True)
                     df.rename(columns={'统计日期': '日期', '商品ID': '商品id'}, inplace=True)
                     shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)', name)[0]
-                    df.insert(loc=1, column='店铺名称', value=shop_name)
+                    if '店铺名称' not in df.columns.tolist():
+                        df.insert(loc=1, column='店铺名称', value=shop_name)
                     new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
-                elif name.endswith('.csv') and '_来源构成_万里马官方旗舰店' in name:
+                elif name.endswith('.csv') and '_来源构成_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
                     self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and ('商品类目属性' in name or '商品主图视频' in name or '商品sku属性' in name):
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                    new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
+                    os.rename(os.path.join(root, name), os.path.join(root, new_name))
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -804,16 +824,13 @@ class DataClean:
                 if name.endswith('.xlsx') and '商品素材_' in name:
                     shop_name = re.findall(r'_([\u4e00-\u9fffA-Za-z]+店)_', name)[0]
                     df = pd.read_excel(os.path.join(root, name), header=0)
-                    df.insert(loc=1, column='店铺名称', value=shop_name)
+                    if '店铺名称' not in df.columns.tolist():
+                        df.insert(loc=1, column='店铺名称', value=shop_name)
                     new_name = f'py_xg_{name}'
                     df.to_excel(os.path.join(upload_path, new_name),
                                 index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
-                    if '官方旗舰店' in name:
-                        db_name = '属性设置3'
-                        collection_name = '商品素材_天猫'
-                    elif '官方企业店' in name:
-                        db_name = '属性设置3'
-                        collection_name = '商品素材_淘宝'
+                    db_name = '属性设置3'
+                    collection_name = '商品素材'
                     os.remove(os.path.join(root, name))
 
                 # 将数据传入 self.datas 等待更新进数据库
@@ -881,11 +898,15 @@ class DataClean:
                 if 'py_xg' not in name:  # 排除非目标文件
                     continue
 
-                if name.endswith('.csv') and '商品排行_万里马官方旗舰店' in name:
-                    t_path = os.path.join(self.source_path, '天猫_生意参谋', '商品排行')
+                if name.endswith('.csv') and '商品排行_' in name:
+                    t_path = os.path.join(self.source_path, '生意参谋', '商品排行')
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '店铺来源_来源构成_' in name:
+                    t_path = os.path.join(self.source_path, '生意参谋', '店铺流量来源')
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '店铺来源_来源构成_万里马官方旗舰店' in name:
-                    t_path = os.path.join(self.source_path, '天猫_生意参谋', '店铺流量来源')
+                elif name.endswith('.csv') and (
+                        '商品类目属性' in name or '商品主图视频' in name or '商品sku属性' in name):
+                    t_path = os.path.join(self.source_path, '生意参谋', '商品属性')
                     bib(t_path, _as_month=True)
 
     def move_dmp(self, path=None, is_except=[]):
@@ -1097,10 +1118,10 @@ class DataClean:
                         t_path = os.path.join(self.source_path, '天猫推广报表', '品销宝', '定向人群报表')
                         bib(t_path, _as_month=True)
                 elif name.endswith('xlsx') and '商品素材_万里马官方旗舰店' in name:
-                    t_path = os.path.join(self.source_path, '商品素材', '天猫')
+                    t_path = os.path.join(self.source_path, '商品素材')
                     bib(t_path, _as_month=True)
                 elif name.endswith('xlsx') and '商品素材_万里马官方企业店' in name:
-                    t_path = os.path.join(self.source_path, '商品素材', '淘宝')
+                    t_path = os.path.join(self.source_path, '商品素材')
                     bib(t_path, _as_month=True)
 
     # @try_except
@@ -1400,7 +1421,7 @@ def main(service_databases=None, is_mysql=False):
             '天猫_推广数据3',
             '淘宝_推广数据3',
             # '市场数据3',
-            '天猫_生意参谋3',
+            '生意参谋3',
             '天猫_生意经3',
             # '淘宝_生意经3',
         ],

mdbq/dataframe/converter.py

@@ -47,7 +47,7 @@ class DataFrameConverter(object):
             try:
                 # 百分比在某些数据库中不兼容, 转换百分比为小数, # 转百分比的列不能含有中文或特殊字符
                 df[col] = df[col].apply(
-                    lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+\.?\d*%', str(x)) else x)
+                    lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+\.?\d*%$', str(x)) else x)
             except Exception as e:
                 print(f'留意错误信息: 位于列 -> {col} -> {e}')
 
@@ -94,5 +94,5 @@ if __name__ == '__main__':
     # print(df['a'].dtype)
     # print(df)
     pattern = '1540%'
-    pattern = re.findall(r'^\d+\.?\d*%', pattern)
+    pattern = re.findall(r'^\d+\.?\d*%$', pattern)
     print(pattern)

{mdbq-2.6.1.dist-info → mdbq-2.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.6.1
+Version: 2.6.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.6.1.dist-info → mdbq-2.6.2.dist-info}/RECORD

@@ -5,11 +5,11 @@ mdbq/aggregation/aggregation.py,sha256=ZBsaQbazp2J9rQDoTvwRNav1O4g3PrZue31_6iM4k
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=gdScrgTAb6RbXHZy1LitX7lggMGn1GTLhkYSgztfwew,4903
-mdbq/aggregation/query_data.py,sha256=xVqQBc0kDnm-1giKmAWoKj_obY4GiipgEXY12NhK2AQ,103635
+mdbq/aggregation/query_data.py,sha256=6hEmrqPiI5yUuQyTP5_w9n9aVoOXutuJK6Z_jRlsV_s,104438
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=kkogtWDHE28gyJ2ZgwCqhygbhIrQ9Wcs-G9tFEiKwQc,76837
+mdbq/clean/clean_upload.py,sha256=t_vUPUuDrWAKxdJO8Ptcu9H6aSGzAgBKg9sur-Mr0oM,77929
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
@@ -20,7 +20,7 @@ mdbq/config/products.py,sha256=hN9UMkM6j76HYMulTYdtr3mOhh9QdpvvrLH14a_mbFY,5980
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=SJLZ96f6QBxnPcaaKDi3UOLNk0b7O6aWLAyDOMiSy80,4312
+mdbq/dataframe/converter.py,sha256=KNHxk3dNw1ycOpcnTg83yHrV9B3pvoYwK3Wc_bzk2NE,4314
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
@@ -42,7 +42,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=3EjeTPbwk_qLGMVqDhBZoEPGfD2oM-SBiODjxLL3A8U,16883
-mdbq-2.6.1.dist-info/METADATA,sha256=AyfNicxlm5jVQSJdDDP1w9H2XCa8LZzpA2VXYfR887Q,245
-mdbq-2.6.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.6.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.6.1.dist-info/RECORD,,
+mdbq-2.6.2.dist-info/METADATA,sha256=YWpQrR1j4lfvAEEea_-hzDw7IblqooNfsPi6JcB_d8M,245
+mdbq-2.6.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.6.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.6.2.dist-info/RECORD,,