mdbq 1.2.9__tar.gz → 1.3.0__tar.gz

{mdbq-1.2.9 → mdbq-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.2.9
+Version: 1.3.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.2.9 → mdbq-1.3.0}/mdbq/aggregation/aggregation.py

@@ -174,14 +174,14 @@ class DatabaseUpdate:
                     db_name = '生意参谋2'
                     if '经营优势' in df['一级来源'].tolist():  # 新版流量
                         if '数据周期' in df.columns.tolist():
-                            collection_name='店铺来源_月数据_新版'
+                            collection_name='店铺来源_月数据'
                         else:
-                            collection_name='店铺来源_日数据_新版'
+                            collection_name='店铺来源_日数据'
                     else:  # 旧版流量
                         if '数据周期' in df.columns.tolist():
-                            collection_name='店铺来源_月数据'
+                            collection_name='店铺来源_月数据_旧版'
                         else:
-                            collection_name='店铺来源_日数据'
+                            collection_name='店铺来源_日数据_旧版'
                 elif name.endswith('.xls') and '生意参谋' in name and '商品_全部' in name:
                     # 店铺商品排行
                     df = pd.read_excel(os.path.join(root, name), header=4)
@@ -358,13 +358,21 @@ class DatabaseUpdate:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 elif name.endswith('.csv') and '市场排行_店铺' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                elif name.endswith('.csv') and '类目洞察_属性分析' in name:
+                elif name.endswith('.csv') and '类目洞察_属性分析_分析明细_商品发现' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                elif name.endswith('.csv') and '类目洞察_属性分析_分析明细_汇总' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                elif name.endswith('.csv') and '类目洞察_价格分析_分析明细_商品发现' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                elif name.endswith('.csv') and '类目洞察_价格分析_分析明细_汇总' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                elif name.endswith('.csv') and '竞店分析-销售分析-关键指标对比' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                elif name.endswith('.csv') and '类目洞察_价格分析' in name:
+                elif name.endswith('.csv') and '竞店分析-销售分析-top商品榜' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                elif name.endswith('.csv') and '竞店分析-销售分析' in name:
+                elif name.endswith('.csv') and '竞店分析-来源分析-入店来源' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                elif name.endswith('.csv') and '竞店分析-来源分析' in name:
+                elif name.endswith('.csv') and '竞店分析-来源分析-入店搜索词' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 # ----------------------- 京东数据处理分界线 -----------------------
                 # ----------------------- 京东数据处理分界线 -----------------------
@@ -983,7 +991,7 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
     # db_name = '生意参谋2'
-    # table_name = '店铺来源_日数据_新版'
+    # table_name = '店铺来源_日数据'
     # upload_dir(
     #     path='/Users/xigua/数据中心/原始文件2/生意参谋/流量来源',
     #     db_name=db_name,

{mdbq-1.2.9 → mdbq-1.3.0}/mdbq/aggregation/query_data.py

@@ -167,7 +167,7 @@ class MysqlDatasQuery:
         }
         df = self.download.data_to_df(
             db_name='生意参谋2',
-            table_name='店铺来源_日数据',
+            table_name='店铺来源_日数据_旧版',
             start_date=start_date,
             end_date=end_date,
             projection=projection,
@@ -320,7 +320,7 @@ class GroupBy:
                 }
             )
             return df
-        elif '店铺来源_日数据' in table_name:
+        elif '店铺来源_日数据_旧版' in table_name:
             return df
         elif '商品id编码表' in table_name:
             df['宝贝id'] = df['宝贝id'].astype(str)
@@ -504,7 +504,7 @@ def data_aggregation(service_databases=[{}]):
     for service_database in service_databases:
         for service_name, database in service_database.items():
             sdq = MysqlDatasQuery(target_service=service_name)  # 实例化数据处理类
-            sdq.months = 0  # 设置数据周期， 1 表示近 2 个月
+            sdq.months = 1  # 设置数据周期， 1 表示近 2 个月
             g = GroupBy()  # 实例化数据聚合类
             # 实例化数据库连接
             username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
@@ -526,7 +526,7 @@ def data_aggregation(service_databases=[{}]):
                 },
                 {
                     '数据库名': '聚合数据',
-                    '集合名': '天猫_店铺来源_日数据',
+                    '集合名': '天猫_店铺来源_日数据_旧版',
                     '唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
                     '数据主体': sdq.dplyd(),
                 },
@@ -552,7 +552,7 @@ def data_aggregation(service_databases=[{}]):
             for items in data_dict:  # 遍历返回结果
                 db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
                 df = g.groupby(df=df, table_name=table_name, is_maximize=True)  # 2. 聚合数据
-                g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
+                # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
                 m.df_to_mysql(
                     df=df,
                     db_name=db_name,

{mdbq-1.2.9 → mdbq-1.3.0}/mdbq/clean/data_clean.py

@@ -269,25 +269,25 @@ class DataClean:
                     if '经营优势' in df['一级来源'].tolist():  # 新版流量
                         if '数据周期' in df.columns.tolist():
                             if self.set_up_to_mogo:
-                                d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_店铺来源_月数据_新版')
+                                d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_店铺来源_月数据')
                             if self.set_up_to_mysql:
-                                m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_店铺来源_月数据_新版')
+                                m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_店铺来源_月数据')
                         else:
                             if self.set_up_to_mogo:
-                                d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_店铺来源_日数据_新版')
+                                d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_店铺来源_日数据')
                             if self.set_up_to_mysql:
-                                m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_店铺来源_日数据_新版')
+                                m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_店铺来源_日数据')
                     else:  # 旧版流量
                         if '数据周期' in df.columns.tolist():
                             if self.set_up_to_mogo:
-                                d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_店铺来源_月数据')
+                                d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_店铺来源_月数据_旧版')
                             if self.set_up_to_mysql:
-                                m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_店铺来源_月数据')
+                                m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_店铺来源_月数据_旧版')
                         else:
                             if self.set_up_to_mogo:
-                                d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_店铺来源_日数据')
+                                d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_店铺来源_日数据_旧版')
                             if self.set_up_to_mysql:
-                                m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_店铺来源_日数据')
+                                m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_店铺来源_日数据_旧版')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xls') and '生意参谋' in name and '商品_全部' in name:
@@ -311,9 +311,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_商品排行')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_商品排行')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_商品排行')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_商品排行')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xls') and '参谋店铺整体日报' in name:
@@ -327,9 +327,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df,db_name='生意参谋数据1', collection_name='生意参谋_自助取数_整体日报')
+                        d.df_to_mongo(df=df,db_name='生意参谋2', collection_name='生意参谋_自助取数_整体日报')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_自助取数_整体日报')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_自助取数_整体日报')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xls') and '参谋每日流量_自助取数_新版' in name:
@@ -353,9 +353,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_自助取数_每日流量')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_自助取数_每日流量')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_自助取数_每日流量')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_自助取数_每日流量')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xls') and '商品sku' in name:
@@ -376,9 +376,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_自助取数_商品sku')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_自助取数_商品sku')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_自助取数_商品sku')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_自助取数_商品sku')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xls') and '参谋店铺流量来源（月）' in name:
@@ -403,9 +403,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_自助取数_店铺流量_月数据')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_自助取数_店铺流量_月数据')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_自助取数_店铺流量_月数据')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_自助取数_店铺流量_月数据')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.csv') and 'baobei' in name:
@@ -561,9 +561,9 @@ class DataClean:
                     df = pd.merge(df_lin, df, how='outer', on=col4)
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_直播间成交订单明细')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_直播间成交订单明细')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_直播间成交订单明细')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_直播间成交订单明细')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xlsx') and '直播间大盘数据' in name:
@@ -582,9 +582,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_直播间大盘数据')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_直播间大盘数据')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_直播间大盘数据')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_直播间大盘数据')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xls') and '直播业绩-成交拆解' in name:
@@ -604,9 +604,9 @@ class DataClean:
                     df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
                     self.save_to_csv(df, root, new_name)  # mysql 可能改变 df 列名，所以在上传 mysql 前保存 csv
                     if self.set_up_to_mogo:
-                        d.df_to_mongo(df=df, db_name='生意参谋数据1', collection_name='生意参谋_直播业绩')
+                        d.df_to_mongo(df=df, db_name='生意参谋2', collection_name='生意参谋_直播业绩')
                     if self.set_up_to_mysql:
-                        m.df_to_mysql(df=df, db_name='生意参谋数据1', tabel_name='生意参谋_直播业绩')
+                        m.df_to_mysql(df=df, db_name='生意参谋2', tabel_name='生意参谋_直播业绩')
                     os.remove(os.path.join(root, name))
 
                 elif name.endswith('.xlsx') and '明星店铺' in name:
@@ -988,11 +988,15 @@ class DataClean:
 
                 if name.endswith('.csv') and '无线店铺流量来源' in name:
                     date01 = re.findall(r'\d{4}-\d{2}-(\d{2})_\d{4}-\d{2}-(\d{2})', name)
+
                     if int(date01[0][1]) - int(date01[0][0]) > 15:
-                        t_path = str(pathlib.Path(self.source_path, '月数据/流量来源'))
+                        t_path = str(pathlib.Path(self.source_path, '月数据/流量来源_旧版'))
                         bib(t_path)
-                    else:
+                    elif '_新版' in name:
                         t_path = str(pathlib.Path(self.source_path, '生意参谋/流量来源'))
+                        bib(t_path)
+                    else:
+                        t_path = str(pathlib.Path(self.source_path, '生意参谋/流量来源_旧版'))
                         bib(t_path, _as_month=True)
                 elif name.endswith('.csv') and '商品_全部' in name:
                     t_path = str(pathlib.Path(self.source_path, '生意参谋/商品排行'))
@@ -1009,20 +1013,38 @@ class DataClean:
                 elif name.endswith('.csv') and '参谋店铺流量来源（月）' in name:
                     t_path = str(pathlib.Path(self.source_path, '月数据/流量来源-自助取数-月数据'))
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '竞店分析-' in name:
-                    t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析'))
-                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '竞店分析' in name and '来源分析-入店来源' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析/来源分析/入店来源'))
+                    bib(t_path, _as_month=False)
+                elif name.endswith('.csv') and '竞店分析' in name and '来源分析-入店搜索词' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析/来源分析/入店搜索词'))
+                    bib(t_path, _as_month=False)
+                elif name.endswith('.csv') and '竞店分析' in name and '销售分析-关键指标对比' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析/销售分析/关键指标对比'))
+                    bib(t_path, _as_month=False)
+                elif name.endswith('.csv') and '竞店分析' in name and '销售分析-top商品榜' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/竞店分析/销售分析/top商品榜'))
+                    bib(t_path, _as_month=False)
                 elif name.endswith('.csv') and '监控店铺数据' in name:
                     t_path = str(pathlib.Path(self.source_path, '市场数据/监控店铺数据'))
                     bib(t_path, _as_month=True)
                 elif name.endswith('.csv') and '监控商品' in name:
                     t_path = str(pathlib.Path(self.source_path, '市场数据/监控商品数据'))
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '竞店分析-流量分析' in name:
-                    t_path = str(pathlib.Path(self.source_path, '市场数据/竞店流量构成'))
+                # elif name.endswith('.csv') and '竞店分析-流量分析' in name:
+                #     t_path = str(pathlib.Path(self.source_path, '市场数据/竞店流量构成'))
+                #     bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '类目洞察' in name and '属性分析_分析明细_汇总' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/类目洞察/属性分析/汇总'))
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '类目洞察' in name and '属性分析_分析明细_商品发现' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/类目洞察/属性分析/商品发现'))
+                    bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '类目洞察' in name and '价格分析_分析明细_汇总' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/类目洞察/价格分析/汇总'))
                     bib(t_path, _as_month=True)
-                elif name.endswith('.csv') and '类目洞察' in name:
-                    t_path = str(pathlib.Path(self.source_path, '市场数据/类目洞察'))
+                elif name.endswith('.csv') and '类目洞察' in name and '价格分析_分析明细_商品发现' in name:
+                    t_path = str(pathlib.Path(self.source_path, '市场数据/类目洞察/价格分析/商品发现'))
                     bib(t_path, _as_month=True)
                 elif name.endswith('.csv') and '市场排行_店铺排行' in name:
                     t_path = str(pathlib.Path(self.source_path, '市场数据/市场二级类目店铺'))

{mdbq-1.2.9 → mdbq-1.3.0}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.2.9
+Version: 1.3.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.2.9 → mdbq-1.3.0}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.2.9',
+      version='1.3.0',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',