mdbq 1.5.9__py3-none-any.whl → 1.6.0__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -42,6 +42,7 @@ class DatabaseUpdate:
     def cleaning(self, is_move=True):
         """
         数据清洗, 返回包含 数据库名, 集合名称, 和 df 主体
+        修改 cleaning 时，要同步 support 下的 标题对照表.csv
         """
         if not os.path.exists(self.path):
             print(f'1.1.0 初始化时传入了不存在的目录: {self.path}')
@@ -108,8 +109,12 @@ class DatabaseUpdate:
                         # df.replace(to_replace=['\\N'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
                         # df.replace(to_replace=[''], value=0, regex=False, inplace=True)
                         # df.fillna(0, inplace=True)
-                        db_name = '推广数据2'
-                        collection_name = f'{tg_name}'
+                        if '省' in df.columns.tolist() and '场景名字' in df.columns.tolist() and '地域报表' in name:
+                            db_name = '推广数据2'
+                            collection_name = f'完整_{tg_name}'
+                        else:
+                            db_name = '推广数据2'
+                            collection_name = f'{tg_name}'
                 if name.endswith('.csv') and '超级直播' in name:
                     # 超级直播
                     df = pd.read_csv(os.path.join(root, name), encoding=encoding, header=0, na_filter=False)
@@ -516,6 +521,13 @@ class DatabaseUpdate:
                         print(f'{name} 报表数据为空')
                         continue
                     df = df[df['缩略图'] != '合计']
+                elif name.endswith('.csv') and '营销概况_全站营销' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=1, na_filter=False)
+                    df = df[(df['日期'] != '日期') & (df['日期'] != '汇总') & (df['日期'] != '0') & (df['花费'] != '0') & (df['花费'] != '0.00')]
+                    df['日期'] = df['日期'].apply(lambda x: f'{str(x)[:4]}-{str(x)[4:6]}-{str(x)[6:8]}')
+                    df.drop("'当前时间'", axis=1, inplace=True)
+                    df.rename(columns={'全站ROI': '全站roi'}, inplace=True)
+                    df.insert(loc=1, column='产品线', value='全站营销')
 
                 # 商品素材，必须保持放在最后处理
                 elif name.endswith('xlsx'):
@@ -836,10 +848,11 @@ class DatabaseUpdate:
     def other_table(self, service_databases=[{'home_lx': 'mysql'}]):
         """ 上传 support 文件夹下的 主推商品.csv """
         support_file = set_support.SetSupport(dirname='support').dirname
-        filename = '主推商品.csv'
+        filename = '主推商品.xlsx'
         if not os.path.isfile(os.path.join(support_file, filename)):
             return
-        df = pd.read_csv(os.path.join(support_file, filename), encoding='utf-8_sig', header=0, na_filter=False)
+        # df = pd.read_csv(os.path.join(support_file, filename), encoding='utf-8_sig', header=0, na_filter=False)
+        df = pd.read_excel(os.path.join(support_file, filename), header=0)
         for service_database in service_databases:
             for service_name, database in service_database.items():
                 username, password, host, port = get_myconf.select_config_values(
@@ -961,6 +974,7 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
     username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
     filename = os.path.basename(file)
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False, float_precision='high')
+    # df.replace(to_replace=[','], value='', regex=True, inplace=True)  # 替换掉特殊字符
     m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
     m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename, df_sql=True, drop_duplicates=False,)
 
@@ -1044,22 +1058,17 @@ if __name__ == '__main__':
     print(username, password, host, port)
     # file_dir(one_file=False)
     # one_file_to_mysql(
-    #     file='/Users/xigua/数据中心/原始文件2/推广报表/品销宝/账户/账户_明星店铺报表_2023-11-13_2023-12-12.csv',
-    #     db_name='推广数据2',
-    #     table_name='品销宝',
-    #     target_service='home_lx',
+    #     file='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_全站营销报表/2024-08/万里马箱包推广1_营销概况_全站营销_2024-08-19_2024-09-02.csv',
+    #     db_name='京东数据2',
+    #     table_name='推广数据_全站营销',
+    #     target_service='company',
     #     database='mysql'
     # )
     # db_name = '生意参谋2'
-    # table_name = '店铺来源_日数据'
+    # table_name = '自助取数_店铺流量_月数据'
     # upload_dir(
-    #     path='/Users/xigua/数据中心/原始文件2/生意参谋/流量来源',
+    #     path='/Users/xigua/数据中心/原始文件2/月数据/流量来源-自助取数-月数据',
     #     db_name=db_name,
     #     collection_name=table_name,
     #     dbs={'mysql': True, 'mongodb': False},
     # )
-
-    # test2()
-
-    dp = DatabaseUpdate(path='')
-    dp.other_table(service_databases=[{'company': 'mysql'}])

mdbq/aggregation/query_data.py

@@ -13,6 +13,9 @@ import platform
 import getpass
 import json
 import os
+
+from sqlalchemy.event import remove
+
 """
 程序用于下载数据库(调用 s_query.py 下载并清洗), 并对数据进行聚合清洗, 不会更新数据库信息;
 
@@ -207,6 +210,7 @@ class MysqlDatasQuery:
             columns_name=['日期', '商品id', '商品白底图', '方版场景图'],
         )
         df = pd.DataFrame(data=data_values)
+
         return df
 
     def dplyd(self):
@@ -290,7 +294,28 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
-
+    def jdqzyx(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '产品线': 1,
+            '花费': 1,
+            '全站roi': 1,
+            '全站交易额': 1,
+            '全站订单行': 1,
+            '全站订单成本': 1,
+            '全站费比': 1,
+            '核心位置展现量': 1,
+            '核心位置点击量': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='京东数据2',
+            table_name='推广数据_全站营销',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
     def sku_sales(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -337,6 +362,7 @@ class GroupBy:
             self.output = os.path.join('数据中心/数据库导出')
         self.data_tgyj = {}  # 推广综合聚合数据表
         self.data_jdtg = {}  # 京东推广数据，聚合数据
+        self.sp_index_datas = pd.DataFrame()  # 商品 id 索引表
 
     @staticmethod
     def try_except(func):  # 在类内部定义一个异常处理方法
@@ -353,6 +379,7 @@ class GroupBy:
     def groupby(self, df, table_name, is_maximize=True):
         """
         self.is_maximize: 是否最大转化数据
+        table_name： 聚合数据库处的名称，不是原始数据库
         """
         if isinstance(df, pd.DataFrame):
             if len(df) == 0:
@@ -361,6 +388,7 @@ class GroupBy:
         else:
             print(f'query_data.groupby函数中 {table_name} 传入的 df 不是 dataframe 结构')
             return pd.DataFrame()
+        # print(table_name)
         if '宝贝主体报表' in table_name:
             df.rename(columns={
                 '场景名字': '营销场景',
@@ -419,6 +447,27 @@ class GroupBy:
                     table_name: df_new,
                 }
             )
+            # df_pic：商品排序索引表, 给 powerbi 中的主推款排序用的,(从上月1号到今天的总花费进行排序)
+            today = datetime.date.today()
+            last_month = today - datetime.timedelta(days=30)
+            if last_month.month == 12:
+                year_my = today.year - 1
+            else:
+                year_my = today.year
+            # 截取 从上月1日 至 今天的花费数据, 推广款式按此数据从高到低排序（商品图+排序）
+            df_pic = df.groupby(['日期', '商品id'], as_index=False).agg({'花费': 'sum'})
+            df_pic = df_pic[~df_pic['商品id'].isin([''])]  # 指定列中删除包含空值的行
+            df_pic = df_pic[(df_pic['日期'] >= f'{year_my}-{last_month.month}-01')]
+            df_pic = df_pic.groupby(['商品id'], as_index=False).agg({'花费': 'sum'})
+            df_pic.sort_values('花费', ascending=False, ignore_index=True, inplace=True)
+            df_pic.reset_index(inplace=True)
+            df_pic['index'] = df_pic['index'] + 100
+            df_pic.rename(columns={'index': '商品索引'}, inplace=True)
+            df_pic_new = pd.merge(df, df_pic, how='left', on=['商品id'])
+            df_pic_new['商品索引'].fillna(1000, inplace=True)
+            self.sp_index_datas = df_pic_new[['商品id', '商品索引']]
+            return df
+        elif '商品索引表' in table_name:
             return df
         elif '人群报表' in table_name:
             df.rename(columns={
@@ -639,7 +688,7 @@ class GroupBy:
                 }
             )
             return df
-        elif '京准通' in table_name:
+        elif '京东_京准通' in table_name and '全站营销' not in table_name:
             df = df.groupby(['日期', '产品线', '触发sku id', '跟单sku id', 'spu id', '花费', '展现数', '点击数'], as_index=False).agg(
                 **{'直接订单行': ('直接订单行', np.max),
                    '直接订单金额': ('直接订单金额', np.max),
@@ -656,6 +705,19 @@ class GroupBy:
                 }
             )
             return df
+        elif '京东_京准通_全站营销' in table_name:
+            df = df.groupby(['日期', '产品线', '花费'], as_index=False).agg(
+                **{'全站roi': ('全站roi', np.max),
+                   '全站交易额': ('全站交易额', np.max),
+                   '全站订单行': ('全站订单行', np.max),
+                   '全站订单成本': ('全站订单成本', np.max),
+                   '全站费比': ('全站费比', np.max),
+                   '核心位置展现量': ('核心位置展现量', np.max),
+                   '核心位置点击量': ('核心位置点击量', np.max),
+                   }
+            )
+            df = df[df['花费'] > 0]
+            return df
         elif '京东_sku_商品明细' in table_name:
             df = df[df['商品id'] != '合计']
             df = df.groupby(['日期', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数'],
@@ -849,9 +911,9 @@ def data_aggregation_one(service_databases=[{}], months=1):
             data_dict = [
                 {
                     '数据库名': '聚合数据',
-                    '集合名': '京东_sku_商品明细',
-                    '唯一主键': ['日期', '商品id', '成交单量'],
-                    '数据主体': sdq.sku_sales(),
+                    '集合名': '京东_京准通_全站营销',
+                    '唯一主键': ['日期', '产品线',],
+                    '数据主体': sdq.jdqzyx(),
                 },
             ]
             ######################################################
@@ -938,6 +1000,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': ['日期', '产品线', '触发sku id', '跟单sku id', '花费', ],
                     '数据主体': sdq.jdjzt(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '京东_京准通_全站营销',
+                    '唯一主键': ['日期', '产品线', '花费'],
+                    '数据主体': sdq.jdqzyx(),
+                },
                 {
                     '数据库名': '聚合数据',
                     '集合名': '京东_sku_商品明细',
@@ -960,6 +1028,17 @@ def data_aggregation(service_databases=[{}], months=1):
             for items in data_dict:  # 遍历返回结果
                 db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
                 df = g.groupby(df=df, table_name=table_name, is_maximize=True)  # 2. 聚合数据
+                if len(g.sp_index_datas) != 0:
+                    # 由推广主体报表，写入一个商品索引表，索引规则：从上月 1 号至今花费从高到低排序
+                    m.df_to_mysql(
+                        df=g.sp_index_datas,
+                        db_name='属性设置2',
+                        table_name='商品索引表',
+                        drop_duplicates=False,
+                        icm_update=['商品id'],
+                        service_database=service_database,
+                    )
+                    g.sp_index_datas = pd.DataFrame()  # 重置，不然下个循环会继续刷入数据库
                 # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
                 m.df_to_mysql(
                     df=df,
@@ -1002,7 +1081,7 @@ def data_aggregation(service_databases=[{}], months=1):
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=1)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

mdbq/clean/data_clean.py

@@ -135,6 +135,9 @@ class DataClean:
                             tm_s_name = pattern[0] + shop_name + date_min + date_max
                             new_root_p = pathlib.Path(self.source_path, '推广报表', tg_name)  # 文件夹，未包括文件名
                             df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
+                            if '省' in df.columns.tolist() and '场景名字' in df.columns.tolist():
+                                new_root_p = pathlib.Path(self.source_path, '推广报表', f'完整_{tg_name}')
+                                tm_s_name = f'完整_{tm_s_name}'
                             self.save_to_csv(df, new_root_p, tm_s_name)
                             if self.set_up_to_mogo:
                                 d.df_to_mongo(df=df, db_name='天猫数据1', collection_name=f'天猫_推广_{tg_name}')
@@ -662,6 +665,16 @@ class DataClean:
                         m.df_to_mysql(df=df, db_name='天猫数据1', tabel_name='万相台_人群洞察')
 
                 # ----------------------- 京东数据处理分界线 -----------------------
+                elif name.endswith('.csv') and '营销概况_全站营销' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=1, na_filter=False)
+                    df = df[(df['日期'] != '日期') & (df['日期'] != '汇总') & (df['日期'] != '0') & (df['花费'] != '0') & (df['花费'] != '0.00')]
+                    df['日期'] = df['日期'].apply(lambda x: f'{str(x)[:4]}-{str(x)[4:6]}-{str(x)[6:8]}')
+                    df.drop("'当前时间'", axis=1, inplace=True)
+                    df.rename(columns={'全站ROI': '全站roi'}, inplace=True)
+                    df.insert(loc=1, column='产品线', value='全站营销')
+                    new_name = re.sub('至', '_', name)
+                    self.save_to_csv(df, root, new_name)
+                    os.remove(os.path.join(root, name))
                 elif name.endswith('.xlsx') and '店铺来源_流量来源' in name:
                     # 京东店铺来源
                     if '按天' not in name:
@@ -1150,6 +1163,9 @@ class DataClean:
                 elif name.endswith('.csv') and '付费广告_行业分析_行业大盘' in name:
                     t_path = str(pathlib.Path(self.source_path, '京东报表/行业大盘_流量排行'))
                     bib(t_path, _as_month=False)
+                elif name.endswith('.csv') and '营销概况_全站营销' in name:
+                    t_path = str(pathlib.Path(self.source_path, '京东报表/JD推广_全站营销报表'))
+                    bib(t_path, _as_month=True)
                 #  京东分界线   ------- 结束标记
 
     def attribute(self, path=None, _str='商品素材导出', ):

mdbq/mysql/mysql.py

@@ -9,7 +9,7 @@ import warnings
 import pymysql
 import numpy as np
 import pandas as pd
-# from sqlalchemy import create_engine
+from sqlalchemy import create_engine
 import os
 import calendar
 from mdbq.config import get_myconf

{mdbq-1.5.9.dist-info → mdbq-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.5.9
+Version: 1.6.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.5.9.dist-info → mdbq-1.6.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=gvfaRx-LnzUKlp_kynSsvAnFGEccMKRxS5zPT6UReiw,60536
+mdbq/aggregation/aggregation.py,sha256=L1ZETxvSY7Ads0qlN5fC5xQrrStMPiNNoDrngJnKAXA,61742
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=axYyNQPfKy3n4ztU4LwOhosdCKVfTxJc2Jgt6VU9_KM,48406
+mdbq/aggregation/query_data.py,sha256=NSVMrndo6QsHnjBq6iWvi7OnLo_Shp1GgszvLc2Fe2Q,52491
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/data_clean.py,sha256=BIzc1XCJjJaZyPT6DCRXRCCRwBaeC5_lER0aqYF1P3M,87778
+mdbq/clean/data_clean.py,sha256=qJqM73cAsd1Kz7KlpCUMBPNfRrbfHeT05zRAH55B54g,89162
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=WCZ92vCJAy6_ZFeOxWL-U9gArIpyga4xts-s1wKsspY,17268
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=6hILGosp2fwTp9qgc9IMFTmNzAz0ZG8rznmPxXHY-eA,43331
+mdbq/mysql/mysql.py,sha256=Fiha5MUqac36UUhLfOoRybhwbRftub9qUBi63wVz1Pc,43329
 mdbq/mysql/s_query.py,sha256=WD_pwgUA9pSQMvNKUxQoHKQf2LgkV8PyQV9Te3AJYs4,8175
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.5.9.dist-info/METADATA,sha256=q-5RlbPJbC9qZtR2VcJiASCXGEAaKkc6yIm7JSqZVCY,245
-mdbq-1.5.9.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.5.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.5.9.dist-info/RECORD,,
+mdbq-1.6.0.dist-info/METADATA,sha256=i6L70MhhwWiPA0iTOnW5i_cNX8TNeFTSEnRyP-hE9x8,245
+mdbq-1.6.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.6.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.6.0.dist-info/RECORD,,

{mdbq-1.5.9.dist-info → mdbq-1.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any