mdbq 2.0.5__tar.gz → 2.0.7__tar.gz

{mdbq-2.0.5 → mdbq-2.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.0.5
+Version: 2.0.7
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.0.5 → mdbq-2.0.7}/mdbq/aggregation/aggregation.py

@@ -87,9 +87,19 @@ class DatabaseUpdate:
                     continue
                 df = pd.DataFrame()  # 初始化 df
                 encoding = self.get_encoding(file_path=os.path.join(root, name))  # 用于处理 csv 文件
-                tg_names = ['营销场景报表', '计划报表', '单元报表', '关键词报表', '人群报表', '主体报表',
-                            '其他主体报表',
-                            '创意报表', '地域报表', '权益报表']
+                tg_names = [
+                    '账户报表',  # 旧版，后来改成 营销场景报表了，C 店还是旧版
+                    '营销场景报表',
+                    '计划报表',
+                    '单元报表',
+                    '关键词报表',
+                    '人群报表',
+                    '主体报表',
+                    '其他主体报表',
+                    '创意报表',
+                    '地域报表',
+                    '权益报表',
+                ]
                 for tg_name in tg_names:
                     if tg_name in name and '报表汇总' not in name and name.endswith('.csv'):  # 排除达摩盘报表: 人群报表汇总
                         pattern = re.findall(r'(.*_)\d{8}_\d{6}', name)
@@ -125,10 +135,16 @@ class DatabaseUpdate:
                         # df.replace(to_replace=[''], value=0, regex=False, inplace=True)
                         # df.fillna(0, inplace=True)
                         if '省' in df.columns.tolist() and '场景名字' in df.columns.tolist() and '地域报表' in name:
-                            db_name = '推广数据2'
+                            if shop_name == '广东万里马':
+                                db_name = '推广数据_淘宝店'
+                            else:
+                                db_name = '推广数据2'
                             collection_name = f'完整_{tg_name}'
                         else:
-                            db_name = '推广数据2'
+                            if shop_name == '广东万里马':
+                                db_name = '推广数据_淘宝店'
+                            else:
+                                db_name = '推广数据2'
                             collection_name = f'{tg_name}'
                         check_remove_file = True
                 if name.endswith('.csv') and '超级直播' in name:
@@ -148,6 +164,8 @@ class DatabaseUpdate:
                         shop_name = ''
                     # df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
                     # df.replace(to_replace=[''], value=0, regex=False, inplace=True)
+                    if shop_name == '广东万里马':
+                        db_name = '推广数据_淘宝店'
                     check_remove_file = True
                 elif name.endswith('.xls') and '短直联投' in name:
                     # 短直联投
@@ -1055,13 +1073,17 @@ class DatabaseUpdate:
         # return df
 
     def other_table(self, service_databases=[{'home_lx': 'mysql'}]):
-        """ 上传 support 文件夹下的 主推商品.csv """
+        """ 上传 support 文件夹下的 主推货品 """
         support_file = set_support.SetSupport(dirname='support').dirname
-        filename = '主推商品.xlsx'
+        filename = '主推货品.xlsx'
         if not os.path.isfile(os.path.join(support_file, filename)):
             return
         # df = pd.read_csv(os.path.join(support_file, filename), encoding='utf-8_sig', header=0, na_filter=False)
         df = pd.read_excel(os.path.join(support_file, filename), header=0)
+        for col in df.columns.tolist():
+            if '预算' in col:
+                df.rename(columns={col: '预算占比'}, inplace=True)
+        df = df[['商品id', '商家编码', '预算占比']]
         for service_database in service_databases:
             for service_name, database in service_database.items():
                 username, password, host, port = get_myconf.select_config_values(
@@ -1077,7 +1099,7 @@ class DatabaseUpdate:
                 m.df_to_mysql(
                     df=df,
                     db_name='属性设置2',
-                    table_name='主推商品',
+                    table_name='主推货品',
                     move_insert=False,  # 先删除，再插入
                     df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
                     drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢

{mdbq-2.0.5 → mdbq-2.0.7}/mdbq/aggregation/optimize_data.py

@@ -57,7 +57,7 @@ def restart_mongodb():
         subprocess.call(command, shell=True)
 
 
-def op_data(db_name_lists, service_databases=None, days: int = 63):
+def op_data(db_name_lists, service_databases=None, days: int = 63, is_mongo=True, is_mysql=True):
     """ service_databases 这个参数暂时没有用 """
     # for service_database in service_databases:
     #     for service_name, database in service_database.items():
@@ -71,42 +71,45 @@ def op_data(db_name_lists, service_databases=None, days: int = 63):
 
     if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
         # mongodb
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
-            database='mongodb',
-        )
-        m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
-        m.db_name_lists = db_name_lists
-        m.days = days
-        m.optimize_list()
-        if m.client:
-            m.client.close()
-            print(f'已关闭 mongodb 连接')
+        if is_mongo:
+            username, password, host, port = get_myconf.select_config_values(
+                target_service='home_lx',
+                database='mongodb',
+            )
+            m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
+            m.db_name_lists = db_name_lists
+            m.days = days
+            m.optimize_list()
+            if m.client:
+                m.client.close()
+                print(f'已关闭 mongodb 连接')
 
-        if socket.gethostname() == 'xigua_lx':
-            restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
+            if socket.gethostname() == 'xigua_lx':
+                restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
 
         # Mysql
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='home_lx',
-            database='mysql',
-        )
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = db_name_lists
-        s.days = days
-        s.optimize_list()
+        if is_mysql:
+            username, password, host, port = get_myconf.select_config_values(
+                target_service='home_lx',
+                database='mysql',
+            )
+            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+            s.db_name_lists = db_name_lists
+            s.days = days
+            s.optimize_list()
 
     elif socket.gethostname() == 'company':
         # Mysql
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='company',
-            database='mysql',
-        )
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = db_name_lists
-        s.days = days
-        s.optimize_list()
+        if is_mysql:
+            username, password, host, port = get_myconf.select_config_values(
+                target_service='company',
+                database='mysql',
+            )
+            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+            s.db_name_lists = db_name_lists
+            s.days = days
+            s.optimize_list()
 
 
 if __name__ == '__main__':
-    op_data(db_name_lists=['聚合数据'], service_databases=[{'company': 'mysql'}], days=3650)
+    op_data(db_name_lists=['聚合数据'], service_databases=[{'company': 'mysql'}], days=3650, is_mongo=True, is_mysql=True)

{mdbq-2.0.5 → mdbq-2.0.7}/mdbq/aggregation/query_data.py

@@ -498,6 +498,173 @@ class MysqlDatasQuery:
         )
         return df
 
+    def tg_by_day(self):
+        """
+        汇总各个店铺的推广数据，按日汇总
+        """
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '场景名字': 1,
+            '花费': 1,
+            '展现量': 1,
+            '点击量': 1,
+            '总购物车数': 1,
+            '总成交笔数': 1,
+            '总成交金额': 1,
+        }
+        df_tm = self.download.data_to_df(
+            db_name='推广数据2',
+            table_name='营销场景报表',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        df_tm.rename(columns={'场景名字': '营销场景'}, inplace=True)
+        df_tm = df_tm.groupby(
+            ['日期', '营销场景', '花费'],
+            as_index=False).agg(
+            **{
+                '展现量': ('展现量', np.max),
+                '点击量': ('点击量', np.max),
+                '加购量': ('总购物车数', np.max),
+                '成交笔数': ('总成交笔数', np.max),
+                '成交金额': ('总成交金额', np.max)
+            }
+        )
+        df_tm.insert(loc=1, column='店铺', value='WLM天猫旗舰店')  # df中添加列
+
+        df_tb = self.download.data_to_df(
+            db_name='推广数据_淘宝店',
+            table_name='账户报表',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        df_tb.rename(columns={'场景名字': '营销场景'}, inplace=True)
+        df_tb = df_tb.groupby(
+            ['日期', '营销场景', '花费'],
+            as_index=False).agg(
+            **{
+                '展现量': ('展现量', np.max),
+                '点击量': ('点击量', np.max),
+                '加购量': ('总购物车数', np.max),
+                '成交笔数': ('总成交笔数', np.max),
+                '成交金额': ('总成交金额', np.max)
+            }
+        )
+        df_tb.insert(loc=1, column='店铺', value='淘宝c店')  # df中添加列
+
+        projection = {
+            '日期': 1,
+            '报表类型': 1,
+            '消耗': 1,
+            '展现量': 1,
+            '点击量': 1,
+            '宝贝加购数': 1,
+            '成交笔数': 1,
+            '成交金额': 1,
+        }
+        df_tm_pxb = self.download.data_to_df(
+            db_name='推广数据2',
+            table_name='品销宝',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        df_tm_pxb = df_tm_pxb[df_tm_pxb['报表类型'] == '账户']
+        df_tm_pxb = df_tm_pxb.groupby(
+            ['日期', '报表类型', '消耗'],
+            as_index=False).agg(
+            **{
+                '展现量': ('展现量', np.max),
+                '点击量': ('点击量', np.max),
+                '加购量': ('宝贝加购数', np.max),
+                '成交笔数': ('成交笔数', np.max),
+                '成交金额': ('成交金额', np.max)
+            }
+        )
+        df_tm_pxb.rename(columns={'报表类型': '营销场景', '消耗': '花费'}, inplace=True)
+        df_tm_pxb['营销场景'] = '品销宝'
+        df_tm_pxb.insert(loc=1, column='店铺', value='WLM天猫旗舰店')  # df中添加列
+
+        projection = {
+            '日期': 1,
+            '产品线': 1,
+            '触发sku id': 1,
+            '跟单sku id': 1,
+            '花费': 1,
+            '展现数': 1,
+            '点击数': 1,
+            '直接订单行': 1,
+            '直接订单金额': 1,
+            '总订单行': 1,
+            '总订单金额': 1,
+            '直接加购数': 1,
+            '总加购数': 1,
+            'spu id': 1,
+        }
+        df_jd = self.download.data_to_df(
+            db_name='京东数据2',
+            table_name='推广数据_京准通',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        df_jd = df_jd.groupby(['日期', '产品线', '触发sku id', '跟单sku id', 'spu id', '花费', '展现数', '点击数'],
+                        as_index=False).agg(
+            **{'直接订单行': ('直接订单行', np.max),
+               '直接订单金额': ('直接订单金额', np.max),
+               '成交笔数': ('总订单行', np.max),
+               '成交金额': ('总订单金额', np.max),
+               '直接加购数': ('直接加购数', np.max),
+               '加购量': ('总加购数', np.max),
+               }
+        )
+        df_jd = df_jd[['日期', '产品线', '花费', '展现数', '点击数', '加购量', '成交笔数', '成交金额']]
+        df_jd.rename(columns={'产品线': '营销场景', '展现数': '展现量', '点击数': '点击量'}, inplace=True)
+        df_jd = df_jd[df_jd['花费'] > 0]
+        df_jd.insert(loc=1, column='店铺', value='京东箱包')  # df中添加列
+
+        projection = {
+            '日期': 1,
+            '产品线': 1,
+            '花费': 1,
+            '全站roi': 1,
+            '全站交易额': 1,
+            '全站订单行': 1,
+            '全站订单成本': 1,
+            '全站费比': 1,
+            '核心位置展现量': 1,
+            '核心位置点击量': 1,
+        }
+        df_jd_qzyx = self.download.data_to_df(
+            db_name='京东数据2',
+            table_name='推广数据_全站营销',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        df_jd_qzyx = df_jd_qzyx.groupby(['日期', '产品线', '花费'], as_index=False).agg(
+            **{'全站roi': ('全站roi', np.max),
+               '成交金额': ('全站交易额', np.max),
+               '成交笔数': ('全站订单行', np.max),
+               '全站订单成本': ('全站订单成本', np.max),
+               '全站费比': ('全站费比', np.max),
+               '展现量': ('核心位置展现量', np.max),
+               '点击量': ('核心位置点击量', np.max),
+               }
+        )
+        df_jd_qzyx.rename(columns={'产品线': '营销场景'}, inplace=True)
+        df_jd_qzyx = df_jd_qzyx[['日期', '营销场景', '花费', '展现量', '点击量', '成交笔数', '成交金额']]
+        df_jd_qzyx = df_jd_qzyx[df_jd_qzyx['花费'] > 0]
+        df_jd_qzyx.insert(loc=1, column='店铺', value='京东箱包')  # df中添加列
+
+        _datas = [item for item in  [df_tm, df_tb, df_tm_pxb, df_jd, df_jd_qzyx] if len(item) > 0]  # 阻止空的 dataframe
+        df = pd.concat(_datas, axis=0, ignore_index=True)
+        return df
+
+
 class GroupBy:
     """
     数据聚合和导出
@@ -540,6 +707,8 @@ class GroupBy:
                     }
                 )
                 return pd.DataFrame()
+        # elif '多店推广场景_按日聚合' in table_name:  # 这个函数传递的是多个 df 组成的列表，暂时放行
+        #     pass
         else:
             print(f'query_data.groupby函数中 {table_name} 传入的 df 不是 dataframe 结构')
             return pd.DataFrame()
@@ -1063,6 +1232,22 @@ class GroupBy:
         elif '直播场次分析' in table_name:
             df.drop_duplicates(subset=['场次id'], keep='first', inplace=True, ignore_index=True)
             return df
+        elif '多店推广场景_按日聚合' in table_name:
+            df = df.groupby(
+                ['日期', '店铺', '营销场景'],
+                as_index=False).agg(
+                **{
+                    '花费': ('花费', np.sum),
+                    '展现量': ('展现量', np.sum),
+                    '点击量': ('点击量', np.sum),
+                    '加购量': ('加购量', np.sum),
+                    '成交笔数': ('成交笔数', np.sum),
+                    '成交金额': ('成交金额', np.sum)
+                }
+            )
+            df.sort_values(['日期', '店铺', '花费'], ascending=[False, False, False], ignore_index=True, inplace=True)
+            # df.to_csv('/Users/xigua/Downloads/test.csv', encoding='utf-8_sig', index=False, header=True)
+            return df
         else:
             print(f'<{table_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
@@ -1437,6 +1622,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': ['场次id'],
                     '数据主体': sdq.zb_ccfx(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '多店推广场景_按日聚合',
+                    '唯一主键': [],
+                    '数据主体': sdq.tg_by_day(),
+                },
             ]
             for items in data_dict:  # 遍历返回结果
                 db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
@@ -1535,7 +1726,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=7)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

{mdbq-2.0.5 → mdbq-2.0.7}/mdbq/clean/data_clean.py

@@ -100,9 +100,19 @@ class DataClean:
                 try:
                     encoding = self.get_encoding(file_path=pathlib.Path(root, name))
                     # ----------------- 推广报表 分割线 -----------------
-                    tg_names = ['营销场景报表', '计划报表', '单元报表', '关键词报表', '人群报表', '主体报表',
-                                '其他主体报表',
-                                '创意报表', '地域报表', '权益报表']
+                    tg_names = [
+                        '账户报表',  # 旧版，后来改成 营销场景报表了，C 店还是旧版
+                        '营销场景报表',
+                        '计划报表',
+                        '单元报表',
+                        '关键词报表',
+                        '人群报表',
+                        '主体报表',
+                        '其他主体报表',
+                        '创意报表',
+                        '地域报表',
+                        '权益报表',
+                    ]
                     for tg_name in tg_names:
                         if tg_name in name and '汇总' not in name and name.endswith('.csv'):  # 人群报表排除达摩盘报表： 人群报表汇总
                             pattern = re.findall(r'(.*_)\d{8}_\d{6}', name)
@@ -153,16 +163,22 @@ class DataClean:
                             date_max = f'{df["日期"].values.max()}.csv'
                             if '万里马' in name:
                                 tm_s_name = pattern[0] + shop_name + date_min + date_max
-                                new_root_p = pathlib.Path(self.source_path, '推广报表', tg_name)  # 文件夹，未包括文件名
+                                if shop_name == '广东万里马':
+                                    new_root_p = pathlib.Path(self.source_path, '推广报表_淘宝店', tg_name)  # 文件夹，未包括文件名
+                                else:
+                                    new_root_p = pathlib.Path(self.source_path, '推广报表', tg_name)  # 文件夹，未包括文件名
                                 df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                                 if '省' in df.columns.tolist() and '场景名字' in df.columns.tolist() and '完整' in name:
-                                    new_root_p = pathlib.Path(self.source_path, '推广报表', f'完整_{tg_name}')
+                                    if shop_name == '广东万里马':
+                                        new_root_p = pathlib.Path(self.source_path, '推广报表_淘宝店', f'完整_{tg_name}')
+                                    else:
+                                        new_root_p = pathlib.Path(self.source_path, '推广报表', f'完整_{tg_name}')
                                     tm_s_name = f'完整_{tm_s_name}'
                                 self.save_to_csv(df, new_root_p, tm_s_name)
-                                if self.set_up_to_mogo:
-                                    d.df_to_mongo(df=df, db_name='天猫数据1', collection_name=f'天猫_推广_{tg_name}')
-                                if self.set_up_to_mysql:
-                                    m.df_to_mysql(df=df, db_name='天猫数据1', tabel_name=f'天猫_推广_{tg_name}')
+                                # if self.set_up_to_mogo:
+                                #     d.df_to_mongo(df=df, db_name='天猫数据1', collection_name=f'天猫_推广_{tg_name}')
+                                # if self.set_up_to_mysql:
+                                #     m.df_to_mysql(df=df, db_name='天猫数据1', tabel_name=f'天猫_推广_{tg_name}')
                                 os.remove(os.path.join(root, name))
                             else:
                                 print(f'{name} 文件名不含"万里马", 不属于爬虫下载，您可以手动进行分类，但不会上传数据库')

{mdbq-2.0.5 → mdbq-2.0.7}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.0.5
+Version: 2.0.7
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.0.5 → mdbq-2.0.7}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.0.5',
+      version='2.0.7',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',