mdbq 2.3.5__py3-none-any.whl → 2.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/query_data.py +166 -1
- mdbq/company/copysh.py +7 -2
- mdbq/req_post/req_tb.py +425 -134
- {mdbq-2.3.5.dist-info → mdbq-2.3.7.dist-info}/METADATA +1 -1
- {mdbq-2.3.5.dist-info → mdbq-2.3.7.dist-info}/RECORD +7 -7
- {mdbq-2.3.5.dist-info → mdbq-2.3.7.dist-info}/WHEEL +0 -0
- {mdbq-2.3.5.dist-info → mdbq-2.3.7.dist-info}/top_level.txt +0 -0
mdbq/aggregation/query_data.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# -*- coding: UTF-8 –*-
|
2
2
|
import re
|
3
|
+
from unittest.mock import inplace
|
3
4
|
|
4
5
|
from mdbq.mongo import mongo
|
5
6
|
from mdbq.mysql import mysql
|
@@ -1003,6 +1004,23 @@ class GroupBy:
|
|
1003
1004
|
df.insert(loc=1, column='推广渠道', value='万相台无界版') # df中插入新列
|
1004
1005
|
df['是否品牌词'] = df['词名字/词包名字'].str.contains('万里马|wanlima', regex=True)
|
1005
1006
|
df['是否品牌词'] = df['是否品牌词'].apply(lambda x: '品牌词' if x else '')
|
1007
|
+
dir_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\分类配置文件.xlsx'
|
1008
|
+
dir_file2 = '/Volumes/时尚事业部/01.运营部/0-电商周报-每周五更新/分类配置文件.xlsx'
|
1009
|
+
if not os.path.isfile(dir_file):
|
1010
|
+
dir_file = dir_file2
|
1011
|
+
if os.path.isfile(dir_file):
|
1012
|
+
df_fl = pd.read_excel(dir_file, sheet_name='关键词分类', header=0)
|
1013
|
+
df_fl.rename(columns={'分类1': '词分类'}, inplace=True)
|
1014
|
+
df_fl = df_fl[['关键词', '词分类']]
|
1015
|
+
df = pd.merge(df, df_fl, left_on=['词名字/词包名字'], right_on=['关键词'], how='left')
|
1016
|
+
df.pop('关键词')
|
1017
|
+
df['词分类'].fillna('', inplace=True)
|
1018
|
+
if '词分类' in df.columns.tolist():
|
1019
|
+
df['词分类'] = df.apply(lambda x: self.ret_keyword(keyword=str(x['词名字/词包名字']), as_file=False) if x['词分类'] == '' else x['词分类'], axis=1)
|
1020
|
+
else:
|
1021
|
+
df['词分类'] = df['词名字/词包名字'].apply(lambda x: self.ret_keyword(keyword=str(x), as_file=False))
|
1022
|
+
# df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
|
1023
|
+
# breakpoint()
|
1006
1024
|
return df
|
1007
1025
|
elif '天猫_超级直播' in table_name:
|
1008
1026
|
df.rename(columns={
|
@@ -1389,6 +1407,153 @@ class GroupBy:
|
|
1389
1407
|
print(f'<{table_name}>: Groupby 类尚未配置,数据为空')
|
1390
1408
|
return pd.DataFrame({})
|
1391
1409
|
|
1410
|
+
def ret_keyword(self, keyword, as_file=False):
|
1411
|
+
datas = [
|
1412
|
+
{
|
1413
|
+
'类别': '品牌词',
|
1414
|
+
'值': [
|
1415
|
+
'万里马',
|
1416
|
+
'wanlima',
|
1417
|
+
'fion',
|
1418
|
+
'菲安妮',
|
1419
|
+
'迪桑娜',
|
1420
|
+
'dissona',
|
1421
|
+
'hr',
|
1422
|
+
'vh',
|
1423
|
+
'songmont',
|
1424
|
+
'vanessahogan',
|
1425
|
+
'dilaks',
|
1426
|
+
'khdesign',
|
1427
|
+
'peco',
|
1428
|
+
'giimmii',
|
1429
|
+
'cassile',
|
1430
|
+
'grotto',
|
1431
|
+
'why',
|
1432
|
+
'roulis',
|
1433
|
+
'lesschic',
|
1434
|
+
'amazing song',
|
1435
|
+
'mytaste',
|
1436
|
+
'bagtree',
|
1437
|
+
'红谷',
|
1438
|
+
'hongu',
|
1439
|
+
]
|
1440
|
+
},
|
1441
|
+
{
|
1442
|
+
'类别': '智选',
|
1443
|
+
'值': [
|
1444
|
+
'智选',
|
1445
|
+
]
|
1446
|
+
},
|
1447
|
+
{
|
1448
|
+
'类别': '年份',
|
1449
|
+
'值': [
|
1450
|
+
'20',
|
1451
|
+
]
|
1452
|
+
},
|
1453
|
+
{
|
1454
|
+
'类别': '材质',
|
1455
|
+
'值': [
|
1456
|
+
'皮',
|
1457
|
+
'牛仔',
|
1458
|
+
'丹宁',
|
1459
|
+
'帆布',
|
1460
|
+
]
|
1461
|
+
},
|
1462
|
+
{
|
1463
|
+
'类别': '季节',
|
1464
|
+
'值': [
|
1465
|
+
'春',
|
1466
|
+
'夏',
|
1467
|
+
'秋',
|
1468
|
+
'冬',
|
1469
|
+
]
|
1470
|
+
},
|
1471
|
+
{
|
1472
|
+
'类别': '款式',
|
1473
|
+
'值': [
|
1474
|
+
'水桶',
|
1475
|
+
'托特',
|
1476
|
+
'腋下',
|
1477
|
+
'小方',
|
1478
|
+
'通用款',
|
1479
|
+
'手拿',
|
1480
|
+
'马鞍',
|
1481
|
+
'链条',
|
1482
|
+
'菜篮',
|
1483
|
+
'hobo',
|
1484
|
+
'波士顿',
|
1485
|
+
'凯莉',
|
1486
|
+
'饺子',
|
1487
|
+
'盒子',
|
1488
|
+
'牛角',
|
1489
|
+
'公文',
|
1490
|
+
'月牙',
|
1491
|
+
'单肩',
|
1492
|
+
'枕头',
|
1493
|
+
'斜挎',
|
1494
|
+
'手提',
|
1495
|
+
'手拎',
|
1496
|
+
'拎手',
|
1497
|
+
'斜肩',
|
1498
|
+
'棒球',
|
1499
|
+
'饺包',
|
1500
|
+
'保龄球',
|
1501
|
+
'戴妃',
|
1502
|
+
'半月',
|
1503
|
+
'弯月',
|
1504
|
+
'法棍',
|
1505
|
+
'流浪',
|
1506
|
+
'拎包',
|
1507
|
+
'中式',
|
1508
|
+
'手挽',
|
1509
|
+
'皮带',
|
1510
|
+
'眼镜',
|
1511
|
+
'斜跨',
|
1512
|
+
'律师',
|
1513
|
+
'斜背',
|
1514
|
+
]
|
1515
|
+
},
|
1516
|
+
{
|
1517
|
+
'类别': '品类词',
|
1518
|
+
'值': [
|
1519
|
+
'老花',
|
1520
|
+
'包包',
|
1521
|
+
'通勤',
|
1522
|
+
'高级感',
|
1523
|
+
'轻奢',
|
1524
|
+
'包',
|
1525
|
+
'新款',
|
1526
|
+
'小众',
|
1527
|
+
'爆款',
|
1528
|
+
'工作',
|
1529
|
+
'精致',
|
1530
|
+
'奢侈',
|
1531
|
+
'袋',
|
1532
|
+
],
|
1533
|
+
},
|
1534
|
+
]
|
1535
|
+
if as_file:
|
1536
|
+
with open(os.path.join(self.output, f'分类配置.json'), 'w') as f:
|
1537
|
+
json.dump(datas, f, ensure_ascii=False, sort_keys=False, indent=4)
|
1538
|
+
breakpoint()
|
1539
|
+
result = ''
|
1540
|
+
res = []
|
1541
|
+
is_continue = False
|
1542
|
+
for data in datas:
|
1543
|
+
for item in data['值']:
|
1544
|
+
if item == '20':
|
1545
|
+
pattern = r'\d\d'
|
1546
|
+
res = re.findall(f'{item}{pattern}', str(keyword), re.IGNORECASE)
|
1547
|
+
else:
|
1548
|
+
res = re.findall(item, str(keyword), re.IGNORECASE)
|
1549
|
+
if res:
|
1550
|
+
result = data['类别']
|
1551
|
+
is_continue = True
|
1552
|
+
break
|
1553
|
+
if is_continue:
|
1554
|
+
break
|
1555
|
+
return result
|
1556
|
+
|
1392
1557
|
# @try_except
|
1393
1558
|
def performance(self, bb_tg=True):
|
1394
1559
|
# print(self.data_tgyj)
|
@@ -1875,6 +2040,6 @@ def main():
|
|
1875
2040
|
|
1876
2041
|
|
1877
2042
|
if __name__ == '__main__':
|
1878
|
-
data_aggregation(service_databases=[{'company': 'mysql'}], months=
|
2043
|
+
data_aggregation(service_databases=[{'company': 'mysql'}], months=0) # 正常的聚合所有数据
|
1879
2044
|
# data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1) # 单独聚合某一个数据库,具体库进函数编辑
|
1880
2045
|
# optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650) # 立即启动对聚合数据的清理工作
|
mdbq/company/copysh.py
CHANGED
@@ -265,8 +265,13 @@ class TbFiles:
|
|
265
265
|
time.sleep(5)
|
266
266
|
|
267
267
|
# 临时加的
|
268
|
-
excel_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\
|
269
|
-
|
268
|
+
# excel_file = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新\\0-WLM_运营周报-1012输出.xlsx'
|
269
|
+
dir_files = f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\0-电商周报-每周五更新'
|
270
|
+
files = os.listdir(dir_files)
|
271
|
+
for file in files:
|
272
|
+
if file.endswith('.xlsx') and '0-WLM_运营周报' in file and '~' not in file and 'baidu' not in file:
|
273
|
+
excel_file = os.path.join(dir_files, file)
|
274
|
+
r.refresh_excel(file=excel_file)
|
270
275
|
|
271
276
|
self.before_max_time = self.check_change() # 重置值, 避免重复同步
|
272
277
|
|
mdbq/req_post/req_tb.py
CHANGED
@@ -26,68 +26,122 @@ class RequestData:
|
|
26
26
|
self.path = None
|
27
27
|
self.filename = None
|
28
28
|
self.is_json_file = False
|
29
|
+
self.df = pd.DataFrame()
|
29
30
|
|
30
|
-
def
|
31
|
+
def qxg_hx_data(self):
|
32
|
+
""" 抢先购 预热期核心指标 """
|
33
|
+
date = datetime.date.today().strftime('%Y-%m-%d')
|
34
|
+
url = (f'https://sycm.taobao.com/datawar/v4/activity/detail/kpi/coreIndex/live.json?'
|
35
|
+
f'activityId=92072444'
|
36
|
+
f'&status=1'
|
37
|
+
f'&dateType=today'
|
38
|
+
f'&dateRange={date}%7C{date}'
|
39
|
+
f'&_=1729216673692'
|
40
|
+
f'&token=0939158d0'
|
41
|
+
)
|
42
|
+
headers = {'User-Agent': ua_sj.get_ua()}
|
43
|
+
cookies = {
|
44
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
45
|
+
# cookies = {}
|
46
|
+
path = '/Users/xigua/Downloads'
|
47
|
+
filename = 'test'
|
48
|
+
result = requests.get(
|
49
|
+
url=url,
|
50
|
+
headers=headers,
|
51
|
+
cookies=cookies,
|
52
|
+
)
|
53
|
+
m_data = json.loads(result.text)
|
54
|
+
# print(m_data)
|
55
|
+
update_time = m_data['data']['updateTime']
|
56
|
+
all_data = m_data['data']['data']
|
57
|
+
timestamp = all_data['statDate']['value'] // 1000 # 毫秒转为秒,不然无法转换时间戳
|
58
|
+
|
59
|
+
datas=[{
|
60
|
+
'日期': time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(timestamp))),
|
61
|
+
'预热加购人数': all_data['preheatCartByrCnt']['value'],
|
62
|
+
'加购转化率': all_data['cartRate']['value'],
|
63
|
+
'预热加购件数': all_data['preheatCartItmCnt']['value'],
|
64
|
+
'预热访客数': all_data['preheatUv']['value'],
|
65
|
+
'收藏转化率': all_data['cltRate']['value'],
|
66
|
+
'预热收藏次数': all_data['preheatCltItmCnt']['value'],
|
67
|
+
'预热收藏人数': all_data['preheatCltByrCnt']['value'],
|
68
|
+
'更新时间': update_time,
|
69
|
+
'促销活动': '2024双11抢先购',
|
70
|
+
'版块': '预热期核心指标',
|
71
|
+
}]
|
72
|
+
df = pd.DataFrame(datas)
|
73
|
+
df = df.astype({
|
74
|
+
'预热加购人数': int,
|
75
|
+
'预热加购件数': int,
|
76
|
+
'预热访客数': int,
|
77
|
+
'预热收藏次数': int,
|
78
|
+
'预热收藏人数': int,
|
79
|
+
'促销活动': str,
|
80
|
+
'版块': str,
|
81
|
+
}, errors='raise')
|
82
|
+
return '活动分析2', '2024双11抢先购预热期核心指标', df # 注意这些是实际数据表名字
|
83
|
+
|
84
|
+
def ys_ll_data(self):
|
31
85
|
""" 活动预售页面 流量来源 """
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
86
|
+
date = datetime.date.today().strftime('%Y%m%d')
|
87
|
+
url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
|
88
|
+
f'dateRange={date}%7C{date}'
|
89
|
+
f'&dateType=today'
|
90
|
+
f'&pageSize=10'
|
91
|
+
f'&page=1'
|
92
|
+
f'&order=desc'
|
93
|
+
f'&orderBy=frontPreheatUv' # 必传参数
|
94
|
+
f'&activityId=94040472' # 关键,必传参数
|
95
|
+
# f'&activityStatus=3'
|
96
|
+
# f'&device=2'
|
97
|
+
# f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
|
98
|
+
# f'&_=1729079731795'
|
99
|
+
# f'&token=7e94ba030'
|
100
|
+
)
|
101
|
+
headers = {
|
102
|
+
# "referer": "https://dmp.taobao.com/index_new.html",
|
103
|
+
'User-Agent': ua_sj.get_ua(),
|
104
|
+
}
|
105
|
+
cookies = {
|
106
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
107
|
+
path = '/Users/xigua/Downloads'
|
108
|
+
filename = 'test'
|
53
109
|
|
54
|
-
self.date = date
|
55
|
-
self.url = url
|
56
|
-
self.headers = headers
|
57
|
-
self.cookies = cookies
|
58
|
-
self.path = path
|
59
|
-
self.filename = filename
|
60
110
|
result = requests.get(
|
61
|
-
|
62
|
-
headers=
|
63
|
-
cookies=
|
111
|
+
url=url,
|
112
|
+
headers=headers,
|
113
|
+
cookies=cookies,
|
64
114
|
)
|
65
115
|
m_data = json.loads(result.text)
|
66
116
|
update_time = m_data['data']['updateTime']
|
67
117
|
# pt_data = data['data']['data'][0] # 平台流量
|
68
118
|
# gg_data = data['data']['data'][1] # 广告流量
|
119
|
+
datas = []
|
69
120
|
for all_data in m_data['data']['data']:
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
121
|
+
datas.append(
|
122
|
+
{
|
123
|
+
'frontPayByrCnt': all_data['frontPayByrCnt']['value'],
|
124
|
+
'一级标识id': all_data['pageId']['value'],
|
125
|
+
'二级标识id': '',
|
126
|
+
'三级标识id': '',
|
127
|
+
'一级来源': all_data['pageName']['value'],
|
128
|
+
'二级来源': '',
|
129
|
+
'三级来源': '',
|
130
|
+
'活动商品访客数(定金期)': all_data['frontPreheatUv']['value'],
|
131
|
+
'定金支付买家数': all_data['frontPayByrCnt']['value'],
|
132
|
+
'定金支付转化率': all_data['frontPayRate']['value'],
|
133
|
+
'日期': all_data['statDateStr']['value'],
|
134
|
+
'更新时间': update_time,
|
135
|
+
'促销活动': '2024双11预售',
|
136
|
+
'版块': '流量来源',
|
137
|
+
}
|
138
|
+
)
|
85
139
|
if 'children' not in all_data.keys(): # 这一句有点多余,因为一级来源必定细分有二级来源
|
86
140
|
continue
|
87
141
|
for children_data in all_data['children']:
|
88
142
|
one_source_id = children_data['pPageId']['value']
|
89
143
|
one_source_name = children_data['pPageName']['value']
|
90
|
-
|
144
|
+
datas.append(
|
91
145
|
{
|
92
146
|
'frontPayByrCnt': children_data['frontPayByrCnt']['value'],
|
93
147
|
'一级标识id': children_data['pPageId']['value'],
|
@@ -102,6 +156,7 @@ class RequestData:
|
|
102
156
|
'日期': children_data['statDateStr']['value'],
|
103
157
|
'更新时间': update_time,
|
104
158
|
'促销活动': '2024双11预售',
|
159
|
+
'版块': '流量来源',
|
105
160
|
}
|
106
161
|
)
|
107
162
|
# print(children_data['children'])
|
@@ -111,7 +166,7 @@ class RequestData:
|
|
111
166
|
for children_children_data in children_data['children']:
|
112
167
|
# print(children_children_data)
|
113
168
|
# print(one_source_name)
|
114
|
-
|
169
|
+
datas.append(
|
115
170
|
{
|
116
171
|
'frontPayByrCnt': children_children_data['frontPayByrCnt']['value'],
|
117
172
|
'一级标识id': one_source_id,
|
@@ -126,14 +181,276 @@ class RequestData:
|
|
126
181
|
'日期': children_children_data['statDateStr']['value'],
|
127
182
|
'更新时间': update_time,
|
128
183
|
'促销活动': '2024双11预售',
|
184
|
+
'版块': '流量来源',
|
129
185
|
}
|
130
186
|
)
|
131
|
-
for item in
|
187
|
+
for item in datas:
|
132
188
|
if item['日期'] != '':
|
133
189
|
item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
|
134
190
|
if self.is_json_file:
|
135
|
-
|
136
|
-
|
191
|
+
if self.path and self.filename:
|
192
|
+
with open(os.path.join(self.path, f'{self.filename}.json'), 'w') as f:
|
193
|
+
json.dump(datas, f, ensure_ascii=False, sort_keys=True, indent=4)
|
194
|
+
else:
|
195
|
+
print(f'尚未指定 self.path/ self.filename')
|
196
|
+
df = pd.DataFrame(datas)
|
197
|
+
df.fillna('0', inplace=True)
|
198
|
+
df = df.astype(
|
199
|
+
{
|
200
|
+
'frontPayByrCnt': int,
|
201
|
+
'一级标识id': str,
|
202
|
+
'二级标识id': str,
|
203
|
+
'三级标识id': str,
|
204
|
+
'一级来源': str,
|
205
|
+
'二级来源': str,
|
206
|
+
'三级来源': str,
|
207
|
+
'活动商品访客数(定金期)': int,
|
208
|
+
'定金支付买家数': int,
|
209
|
+
'促销活动': str,
|
210
|
+
'版块': str,
|
211
|
+
}, errors='raise')
|
212
|
+
return '活动分析2', '2024双11预售实时流量分析', df # 注意这些是实际数据表名字
|
213
|
+
|
214
|
+
def qxg_ll(self):
|
215
|
+
flow_biz_types = {
|
216
|
+
'classic': '非全站推广期',
|
217
|
+
'qzt': '全站推广期',
|
218
|
+
}
|
219
|
+
page_types = {
|
220
|
+
'item': '商品流量',
|
221
|
+
'shop': '店铺流量',
|
222
|
+
'live': '直播流量',
|
223
|
+
'content': '内容流量',
|
224
|
+
}
|
225
|
+
for k_flow, v_flow in flow_biz_types.items():
|
226
|
+
for k_page, v_page in page_types.items():
|
227
|
+
if v_flow == '全站推广期' and v_page != '商品流量':
|
228
|
+
continue # 只有商品流量才可以传 qzt值
|
229
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
230
|
+
print(f'{now} {v_flow} -> {v_page}: 正在获取数据...')
|
231
|
+
date = datetime.date.today().strftime('%Y%m%d')
|
232
|
+
url = (
|
233
|
+
f'https://sycm.taobao.com/flow/v5/live/shop/source/tree/v4.json?'
|
234
|
+
f'dateRange={date}%7C{date}'
|
235
|
+
f'&dateType=today'
|
236
|
+
f'&order=desc'
|
237
|
+
f'&orderBy=uv'
|
238
|
+
f'&flowBizType={k_flow}' # classic: 非全站推广期,qzt: 全站推广期(只有商品流量才可以传 qzt值)
|
239
|
+
f'&pageType={k_page}' # item:商品流量,shop: 店铺流量,live: 直播流量, content: 内容流量
|
240
|
+
f'&crowdType=all'
|
241
|
+
f'&activityId=92072444'
|
242
|
+
f'&indexCode=uv'
|
243
|
+
# f'&_=1729232086296'
|
244
|
+
# f'&token=2507b8098'
|
245
|
+
)
|
246
|
+
headers = {
|
247
|
+
# "referer": "https://dmp.taobao.com/index_new.html",
|
248
|
+
'User-Agent': ua_sj.get_ua(),
|
249
|
+
}
|
250
|
+
cookies = {
|
251
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
252
|
+
self.qxg_ll_data(
|
253
|
+
url=url,
|
254
|
+
headers=headers,
|
255
|
+
cookies=cookies,
|
256
|
+
flow_biz_type=v_flow,
|
257
|
+
page_type=v_page,
|
258
|
+
)
|
259
|
+
time.sleep(random.randint(5, 10))
|
260
|
+
df = pd.concat(self.datas)
|
261
|
+
df.fillna(0, inplace=True)
|
262
|
+
df = df.astype(
|
263
|
+
{
|
264
|
+
'支付买家数': int,
|
265
|
+
'详情页访客数': int,
|
266
|
+
'来源等级': int,
|
267
|
+
'showDetailChannel': int,
|
268
|
+
}, errors='raise')
|
269
|
+
# df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
|
270
|
+
return '活动分析2', '2024双11抢先购预热期流量来源', df # 注意这些是实际数据表名字
|
271
|
+
|
272
|
+
def qxg_ll_data(self, url, headers, cookies, flow_biz_type, page_type):
|
273
|
+
""" 抢先购 流量来源 """
|
274
|
+
result = requests.get(
|
275
|
+
url=url,
|
276
|
+
headers=headers,
|
277
|
+
cookies=cookies,
|
278
|
+
)
|
279
|
+
json_datas = json.loads(result.text)
|
280
|
+
update_time = json_datas['data']['updateTime']
|
281
|
+
# print(update_time)
|
282
|
+
datas = []
|
283
|
+
json_datas = json_datas['data']['data']
|
284
|
+
dict_data = {}
|
285
|
+
if page_type == '直播流量' or page_type == '内容流量':
|
286
|
+
for item in json_datas:
|
287
|
+
datas.append(
|
288
|
+
{
|
289
|
+
'访客数': item['uv']['value'],
|
290
|
+
'pageId': item['pageId']['value'],
|
291
|
+
'0级来源': item['pageName']['value'],
|
292
|
+
'pPageId': item['pPageId']['value'],
|
293
|
+
'日期': update_time,
|
294
|
+
'更新时间': update_time,
|
295
|
+
'促销活动': '2024双11抢先购',
|
296
|
+
'版块': '流量来源',
|
297
|
+
'来源分类': flow_biz_type,
|
298
|
+
'流量类型': page_type,
|
299
|
+
})
|
300
|
+
json_datas = json_datas[0]['children']
|
301
|
+
|
302
|
+
for all_data in json_datas:
|
303
|
+
# one_source_id = all_data['pageId']['value']
|
304
|
+
one_source_name = all_data['pageName']['value']
|
305
|
+
# print(all_data)
|
306
|
+
for k_first, v_first in all_data.items():
|
307
|
+
# print(k_first, v_first)
|
308
|
+
|
309
|
+
if k_first == 'children':
|
310
|
+
continue
|
311
|
+
for k_second, v_second in v_first.items():
|
312
|
+
if k_second != 'value':
|
313
|
+
dict_data.update({k_second: v_second})
|
314
|
+
dict_data.update(
|
315
|
+
{
|
316
|
+
'guideToShortVideoUv': all_data['guideToShortVideoUv']['value'],
|
317
|
+
'hiddenIndexgroup': all_data['hiddenIndexgroup']['value'],
|
318
|
+
'访客数': all_data['uv']['value'],
|
319
|
+
'访客数占比': all_data['uv']['ratio'],
|
320
|
+
'支付买家数': all_data['payByrCnt']['value'],
|
321
|
+
'详情页访客数': all_data['ipvUvRelate']['value'],
|
322
|
+
'支付转化率': all_data['payRate']['value'],
|
323
|
+
'orderByrCnt': all_data['orderByrCnt']['value'],
|
324
|
+
'showDesc': all_data['showDesc']['value'],
|
325
|
+
'showChannel': all_data['showChannel']['value'],
|
326
|
+
'来源等级': all_data['pageLevel']['value'],
|
327
|
+
'channelType': all_data['channelType']['value'],
|
328
|
+
'orderAmt': all_data['orderAmt']['value'],
|
329
|
+
'pageId': all_data['pageId']['value'],
|
330
|
+
'pPageId': all_data['pPageId']['value'],
|
331
|
+
'payAmt': all_data['payAmt']['value'],
|
332
|
+
'一级来源': all_data['pageName']['value'],
|
333
|
+
'二级来源': '',
|
334
|
+
'三级来源': '',
|
335
|
+
'showDetailChannel': all_data['showDetailChannel']['value'],
|
336
|
+
'pageDesc': all_data['pageDesc']['value'],
|
337
|
+
'payPct': all_data['payPct']['value'],
|
338
|
+
'pPageId': all_data['pPageId']['value'],
|
339
|
+
'crtRate': all_data['crtRate']['value'],
|
340
|
+
'日期': update_time,
|
341
|
+
'更新时间': update_time,
|
342
|
+
'促销活动': '2024双11抢先购',
|
343
|
+
'版块': '流量来源',
|
344
|
+
'来源分类': flow_biz_type,
|
345
|
+
'流量类型': page_type,
|
346
|
+
}
|
347
|
+
)
|
348
|
+
datas.append(dict_data)
|
349
|
+
|
350
|
+
if 'children' not in all_data.keys(): # 这一句有点多余,因为一级来源必定细分有二级来源
|
351
|
+
continue
|
352
|
+
|
353
|
+
for children_data in all_data['children']:
|
354
|
+
# one_source_id = children_data['pPageId']['value']
|
355
|
+
second_source_name = children_data['pageName']['value']
|
356
|
+
for k_first, v_first in children_data.items():
|
357
|
+
# print(k_first, v_first)
|
358
|
+
dict_data = {}
|
359
|
+
if k_first == 'children':
|
360
|
+
continue
|
361
|
+
for k_second, v_second in v_first.items():
|
362
|
+
if k_second != 'value':
|
363
|
+
dict_data.update({k_second: v_second})
|
364
|
+
dict_data.update(
|
365
|
+
{
|
366
|
+
'guideToShortVideoUv': children_data['guideToShortVideoUv']['value'],
|
367
|
+
'hiddenIndexgroup': children_data['hiddenIndexgroup']['value'],
|
368
|
+
'访客数': children_data['uv']['value'],
|
369
|
+
'访客数占比': children_data['uv']['ratio'],
|
370
|
+
'支付买家数': children_data['payByrCnt']['value'],
|
371
|
+
'详情页访客数': children_data['ipvUvRelate']['value'],
|
372
|
+
'支付转化率': children_data['payRate']['value'],
|
373
|
+
'orderByrCnt': children_data['orderByrCnt']['value'],
|
374
|
+
'showDesc': children_data['showDesc']['value'],
|
375
|
+
'showChannel': children_data['showChannel']['value'],
|
376
|
+
'来源等级': children_data['pageLevel']['value'],
|
377
|
+
'channelType': children_data['channelType']['value'],
|
378
|
+
'orderAmt': children_data['orderAmt']['value'],
|
379
|
+
'pageId': children_data['pageId']['value'],
|
380
|
+
'pPageId': children_data['pPageId']['value'],
|
381
|
+
'payAmt': children_data['payAmt']['value'],
|
382
|
+
'一级来源': one_source_name,
|
383
|
+
'二级来源': children_data['pageName']['value'],
|
384
|
+
'三级来源': '',
|
385
|
+
'showDetailChannel': children_data['showDetailChannel']['value'],
|
386
|
+
'pageDesc': children_data['pageDesc']['value'],
|
387
|
+
'payPct': children_data['payPct']['value'],
|
388
|
+
'pPageId': children_data['pPageId']['value'],
|
389
|
+
'crtRate': children_data['crtRate']['value'],
|
390
|
+
'日期': update_time,
|
391
|
+
'更新时间': update_time,
|
392
|
+
'促销活动': '2024双11抢先购',
|
393
|
+
'版块': '流量来源',
|
394
|
+
'来源分类': flow_biz_type,
|
395
|
+
'流量类型': page_type,
|
396
|
+
}
|
397
|
+
)
|
398
|
+
datas.append(dict_data)
|
399
|
+
# print(children_data['children'])
|
400
|
+
# print(children_data)
|
401
|
+
if 'children' not in children_data.keys(): # 部分二级来源没有细分的三级来源,因为需要跳过 children 字段
|
402
|
+
continue
|
403
|
+
for children_children_data in children_data['children']:
|
404
|
+
# print(children_children_data)
|
405
|
+
# print(one_source_name)
|
406
|
+
for k_first, v_first in children_data.items():
|
407
|
+
# print(k_first, v_first)
|
408
|
+
dict_data = {}
|
409
|
+
if k_first == 'children':
|
410
|
+
continue
|
411
|
+
for k_second, v_second in v_first.items():
|
412
|
+
if k_second != 'value':
|
413
|
+
dict_data.update({k_second: v_second})
|
414
|
+
dict_data.update(
|
415
|
+
{
|
416
|
+
'guideToShortVideoUv': children_children_data['guideToShortVideoUv']['value'],
|
417
|
+
'hiddenIndexgroup': children_children_data['hiddenIndexgroup']['value'],
|
418
|
+
'访客数': children_children_data['uv']['value'],
|
419
|
+
'访客数占比': children_children_data['uv']['ratio'],
|
420
|
+
'支付买家数': children_children_data['payByrCnt']['value'],
|
421
|
+
'详情页访客数': children_children_data['ipvUvRelate']['value'],
|
422
|
+
'支付转化率': children_children_data['payRate']['value'],
|
423
|
+
'orderByrCnt': children_children_data['orderByrCnt']['value'],
|
424
|
+
'showDesc': children_children_data['showDesc']['value'],
|
425
|
+
'showChannel': children_children_data['showChannel']['value'],
|
426
|
+
'来源等级': children_children_data['pageLevel']['value'],
|
427
|
+
'channelType': children_children_data['channelType']['value'],
|
428
|
+
'orderAmt': children_children_data['orderAmt']['value'],
|
429
|
+
'pageId': children_children_data['pageId']['value'],
|
430
|
+
'pPageId': children_children_data['pPageId']['value'],
|
431
|
+
'payAmt': children_children_data['payAmt']['value'],
|
432
|
+
'一级来源': one_source_name,
|
433
|
+
'二级来源': second_source_name,
|
434
|
+
'三级来源': children_children_data['pageName']['value'],
|
435
|
+
'showDetailChannel': children_children_data['showDetailChannel']['value'],
|
436
|
+
'pageDesc': children_children_data['pageDesc']['value'],
|
437
|
+
'payPct': children_children_data['payPct']['value'],
|
438
|
+
'pPageId': children_children_data['pPageId']['value'],
|
439
|
+
'crtRate': children_children_data['crtRate']['value'],
|
440
|
+
'日期': update_time,
|
441
|
+
'更新时间': update_time,
|
442
|
+
'促销活动': '2024双11抢先购',
|
443
|
+
'版块': '流量来源',
|
444
|
+
'来源分类': flow_biz_type,
|
445
|
+
'流量类型': page_type,
|
446
|
+
}
|
447
|
+
)
|
448
|
+
datas.append(dict_data)
|
449
|
+
# for item in datas:
|
450
|
+
# if item['日期'] != '':
|
451
|
+
# item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
|
452
|
+
df = pd.DataFrame(datas)
|
453
|
+
self.datas.append(df)
|
137
454
|
|
138
455
|
def hd_sp(self, date, url, headers, cookies, path, filename, pages=5):
|
139
456
|
""" 活动预售页面 分商品效果 """
|
@@ -203,82 +520,6 @@ class RequestData:
|
|
203
520
|
print(result.text)
|
204
521
|
|
205
522
|
|
206
|
-
def tb_data(service_databases=[], db_name=None, table_name=None):
|
207
|
-
""" 2024双11预售实时流量分析 """
|
208
|
-
date = datetime.date.today().strftime('%Y%m%d')
|
209
|
-
url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
|
210
|
-
f'dateRange={date}%7C{date}'
|
211
|
-
f'&dateType=today'
|
212
|
-
f'&pageSize=10'
|
213
|
-
f'&page=1'
|
214
|
-
f'&order=desc'
|
215
|
-
f'&orderBy=frontPreheatUv' # 必传参数
|
216
|
-
f'&activityId=94040472' # 关键,必传参数
|
217
|
-
# f'&activityStatus=3'
|
218
|
-
# f'&device=2'
|
219
|
-
# f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
|
220
|
-
# f'&_=1729079731795'
|
221
|
-
# f'&token=7e94ba030'
|
222
|
-
)
|
223
|
-
headers = {
|
224
|
-
"referer": "https://dmp.taobao.com/index_new.html",
|
225
|
-
'User-Agent': ua_sj.get_ua(),
|
226
|
-
}
|
227
|
-
cookies = {
|
228
|
-
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
229
|
-
path = '/Users/xigua/Downloads'
|
230
|
-
filename = 'test'
|
231
|
-
r = RequestData()
|
232
|
-
r.is_json_file = False
|
233
|
-
r.request_data(
|
234
|
-
date=date,
|
235
|
-
url=url,
|
236
|
-
headers=headers,
|
237
|
-
cookies=cookies,
|
238
|
-
path=path,
|
239
|
-
filename=filename,
|
240
|
-
)
|
241
|
-
# print(r.datas)
|
242
|
-
df = pd.DataFrame(r.datas)
|
243
|
-
# df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
|
244
|
-
|
245
|
-
if not service_databases:
|
246
|
-
return
|
247
|
-
if not db_name or not table_name:
|
248
|
-
print(f'尚未指定 db_name/table_name 参数')
|
249
|
-
return
|
250
|
-
for dt in service_databases:
|
251
|
-
for service_name, database in dt.items():
|
252
|
-
username, password, host, port = get_myconf.select_config_values(
|
253
|
-
target_service=service_name,
|
254
|
-
database=database,
|
255
|
-
)
|
256
|
-
m = mysql.MysqlUpload(
|
257
|
-
username=username,
|
258
|
-
password=password,
|
259
|
-
host=host,
|
260
|
-
port=port,
|
261
|
-
)
|
262
|
-
m.df_to_mysql(
|
263
|
-
df=df,
|
264
|
-
db_name=db_name,
|
265
|
-
table_name=table_name,
|
266
|
-
move_insert=False, # 先删除,再插入
|
267
|
-
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
268
|
-
drop_duplicates=False, # 值为 True 时检查重复数据再插入,反之直接上传,会比较慢
|
269
|
-
filename=None, # 用来追踪处理进度
|
270
|
-
service_database=dt, # 字典
|
271
|
-
)
|
272
|
-
|
273
|
-
|
274
|
-
def company_run():
|
275
|
-
if platform.system() == 'Windows' and socket.gethostname() == 'company':
|
276
|
-
while True:
|
277
|
-
tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2',
|
278
|
-
table_name='2024双11预售实时流量分析')
|
279
|
-
time.sleep(random.randint(1500, 2000))
|
280
|
-
|
281
|
-
|
282
523
|
def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
|
283
524
|
""" 2024双11预售 分商品效果 """
|
284
525
|
date = datetime.date.today().strftime('%Y%m%d')
|
@@ -320,11 +561,61 @@ def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
|
|
320
561
|
df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
|
321
562
|
|
322
563
|
|
564
|
+
def company_run(service_databases=[]):
|
565
|
+
# if platform.system() != 'Windows':
|
566
|
+
# return
|
567
|
+
# if socket.gethostname() != 'company':
|
568
|
+
# return
|
569
|
+
while True:
|
570
|
+
r = RequestData()
|
571
|
+
r.is_json_file = False
|
572
|
+
|
573
|
+
my_data_list = [
|
574
|
+
r.ys_ll_data(), # 双 11预售实时流量分析
|
575
|
+
r.qxg_hx_data(), # 抢先购 预热期核心指标
|
576
|
+
r.qxg_ll() # 抢先购 流量来源
|
577
|
+
]
|
578
|
+
|
579
|
+
results = []
|
580
|
+
for my_data in my_data_list:
|
581
|
+
db_name, table_name, df = my_data
|
582
|
+
if len(df) == 0:
|
583
|
+
print(f'{db_name} -> {table_name} has no data')
|
584
|
+
continue
|
585
|
+
# print(df)
|
586
|
+
results.append([db_name, table_name, df])
|
587
|
+
|
588
|
+
if not service_databases:
|
589
|
+
return
|
590
|
+
for dt in service_databases:
|
591
|
+
for service_name, database in dt.items():
|
592
|
+
username, password, host, port = get_myconf.select_config_values(
|
593
|
+
target_service=service_name,
|
594
|
+
database=database,
|
595
|
+
)
|
596
|
+
m = mysql.MysqlUpload(
|
597
|
+
username=username,
|
598
|
+
password=password,
|
599
|
+
host=host,
|
600
|
+
port=port,
|
601
|
+
)
|
602
|
+
for result in results:
|
603
|
+
db_name, table_name, df = result
|
604
|
+
m.df_to_mysql(
|
605
|
+
df=df,
|
606
|
+
db_name=db_name,
|
607
|
+
table_name=table_name,
|
608
|
+
move_insert=False, # 先删除,再插入
|
609
|
+
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
610
|
+
drop_duplicates=False, # 值为 True 时检查重复数据再插入,反之直接上传,会比较慢
|
611
|
+
filename=None, # 用来追踪处理进度
|
612
|
+
service_database=dt, # 字典
|
613
|
+
)
|
614
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
615
|
+
print(f'{now} {db_name} -> {table_name}: 已入库')
|
616
|
+
|
617
|
+
time.sleep(random.randint(1500, 2000))
|
618
|
+
|
619
|
+
|
323
620
|
if __name__ == '__main__':
|
324
|
-
company_run()
|
325
|
-
# tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2', table_name='2024双11预售实时流量分析')
|
326
|
-
hd_sp_data(
|
327
|
-
service_databases=[{'company': 'mysql'}],
|
328
|
-
# db_name='生意参谋2',
|
329
|
-
# table_name='2024双11预售实时流量分析',
|
330
|
-
)
|
621
|
+
company_run(service_databases=[{'company': 'mysql'}])
|
@@ -5,13 +5,13 @@ mdbq/aggregation/aggregation.py,sha256=CS_gMBwPKQo7uId8BrsYNTjtCZKzRVV7gT4PfE2Q4
|
|
5
5
|
mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
|
6
6
|
mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
|
7
7
|
mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
|
8
|
-
mdbq/aggregation/query_data.py,sha256
|
8
|
+
mdbq/aggregation/query_data.py,sha256=sOsDzZM5C2w82SgCNbJGKYIvHNDmTb-NSQYGdJYjP10,93287
|
9
9
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
10
10
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
11
11
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
12
12
|
mdbq/clean/data_clean.py,sha256=QvbM_mRHvUbwvJu0UpJu4SUQNxLWsFl7QApoq6cPiVU,104788
|
13
13
|
mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
|
14
|
-
mdbq/company/copysh.py,sha256=
|
14
|
+
mdbq/company/copysh.py,sha256=8LL9uPkY0L3XGxTM8G7tROaQX3YzDq5vaE8U1FLe_n0,19714
|
15
15
|
mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
|
16
16
|
mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
17
17
|
mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
|
@@ -38,10 +38,10 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
|
|
38
38
|
mdbq/pbix/refresh_all.py,sha256=viOlLCmz9zg61Q2nzjgl8dChfQxnxRd1A_jmQMb2oDM,5918
|
39
39
|
mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
|
40
40
|
mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
41
|
-
mdbq/req_post/req_tb.py,sha256=
|
41
|
+
mdbq/req_post/req_tb.py,sha256=0gVmmVewIpsJVEXkEEW0UeSSPqE9iAyLpMS0P4qjlX0,35270
|
42
42
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
43
43
|
mdbq/spider/aikucun.py,sha256=fszwS5jml5-S98u5FurBKVS2L4O-0wXFdU6CsGqbWrE,14104
|
44
|
-
mdbq-2.3.
|
45
|
-
mdbq-2.3.
|
46
|
-
mdbq-2.3.
|
47
|
-
mdbq-2.3.
|
44
|
+
mdbq-2.3.7.dist-info/METADATA,sha256=q_SXglctvCzavKdB__xkwQQI4v22pXvtB6cc1YDl8Jk,245
|
45
|
+
mdbq-2.3.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
46
|
+
mdbq-2.3.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
47
|
+
mdbq-2.3.7.dist-info/RECORD,,
|
File without changes
|
File without changes
|