mdbq 2.3.5__py3-none-any.whl → 2.3.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mdbq/req_post/req_tb.py
CHANGED
|
@@ -26,68 +26,122 @@ class RequestData:
|
|
|
26
26
|
self.path = None
|
|
27
27
|
self.filename = None
|
|
28
28
|
self.is_json_file = False
|
|
29
|
+
self.df = pd.DataFrame()
|
|
29
30
|
|
|
30
|
-
def
|
|
31
|
+
def qxg_hx_data(self):
|
|
32
|
+
""" 抢先购 预热期核心指标 """
|
|
33
|
+
date = datetime.date.today().strftime('%Y-%m-%d')
|
|
34
|
+
url = (f'https://sycm.taobao.com/datawar/v4/activity/detail/kpi/coreIndex/live.json?'
|
|
35
|
+
f'activityId=92072444'
|
|
36
|
+
f'&status=1'
|
|
37
|
+
f'&dateType=today'
|
|
38
|
+
f'&dateRange={date}%7C{date}'
|
|
39
|
+
f'&_=1729216673692'
|
|
40
|
+
f'&token=0939158d0'
|
|
41
|
+
)
|
|
42
|
+
headers = {'User-Agent': ua_sj.get_ua()}
|
|
43
|
+
cookies = {
|
|
44
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
|
45
|
+
# cookies = {}
|
|
46
|
+
path = '/Users/xigua/Downloads'
|
|
47
|
+
filename = 'test'
|
|
48
|
+
result = requests.get(
|
|
49
|
+
url=url,
|
|
50
|
+
headers=headers,
|
|
51
|
+
cookies=cookies,
|
|
52
|
+
)
|
|
53
|
+
m_data = json.loads(result.text)
|
|
54
|
+
# print(m_data)
|
|
55
|
+
update_time = m_data['data']['updateTime']
|
|
56
|
+
all_data = m_data['data']['data']
|
|
57
|
+
timestamp = all_data['statDate']['value'] // 1000 # 毫秒转为秒，不然无法转换时间戳
|
|
58
|
+
|
|
59
|
+
datas=[{
|
|
60
|
+
'日期': time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(timestamp))),
|
|
61
|
+
'预热加购人数': all_data['preheatCartByrCnt']['value'],
|
|
62
|
+
'加购转化率': all_data['cartRate']['value'],
|
|
63
|
+
'预热加购件数': all_data['preheatCartItmCnt']['value'],
|
|
64
|
+
'预热访客数': all_data['preheatUv']['value'],
|
|
65
|
+
'收藏转化率': all_data['cltRate']['value'],
|
|
66
|
+
'预热收藏次数': all_data['preheatCltItmCnt']['value'],
|
|
67
|
+
'预热收藏人数': all_data['preheatCltByrCnt']['value'],
|
|
68
|
+
'更新时间': update_time,
|
|
69
|
+
'促销活动': '2024双11抢先购',
|
|
70
|
+
'版块': '预热期核心指标',
|
|
71
|
+
}]
|
|
72
|
+
df = pd.DataFrame(datas)
|
|
73
|
+
df = df.astype({
|
|
74
|
+
'预热加购人数': int,
|
|
75
|
+
'预热加购件数': int,
|
|
76
|
+
'预热访客数': int,
|
|
77
|
+
'预热收藏次数': int,
|
|
78
|
+
'预热收藏人数': int,
|
|
79
|
+
'促销活动': str,
|
|
80
|
+
'版块': str,
|
|
81
|
+
}, errors='raise')
|
|
82
|
+
return '活动分析2', '2024双11抢先购预热期核心指标', df # 注意这些是实际数据表名字
|
|
83
|
+
|
|
84
|
+
def ys_ll_data(self):
|
|
31
85
|
""" 活动预售页面 流量来源 """
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
86
|
+
date = datetime.date.today().strftime('%Y%m%d')
|
|
87
|
+
url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
|
|
88
|
+
f'dateRange={date}%7C{date}'
|
|
89
|
+
f'&dateType=today'
|
|
90
|
+
f'&pageSize=10'
|
|
91
|
+
f'&page=1'
|
|
92
|
+
f'&order=desc'
|
|
93
|
+
f'&orderBy=frontPreheatUv' # 必传参数
|
|
94
|
+
f'&activityId=94040472' # 关键，必传参数
|
|
95
|
+
# f'&activityStatus=3'
|
|
96
|
+
# f'&device=2'
|
|
97
|
+
# f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
|
|
98
|
+
# f'&_=1729079731795'
|
|
99
|
+
# f'&token=7e94ba030'
|
|
100
|
+
)
|
|
101
|
+
headers = {
|
|
102
|
+
# "referer": "https://dmp.taobao.com/index_new.html",
|
|
103
|
+
'User-Agent': ua_sj.get_ua(),
|
|
104
|
+
}
|
|
105
|
+
cookies = {
|
|
106
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
|
107
|
+
path = '/Users/xigua/Downloads'
|
|
108
|
+
filename = 'test'
|
|
53
109
|
|
|
54
|
-
self.date = date
|
|
55
|
-
self.url = url
|
|
56
|
-
self.headers = headers
|
|
57
|
-
self.cookies = cookies
|
|
58
|
-
self.path = path
|
|
59
|
-
self.filename = filename
|
|
60
110
|
result = requests.get(
|
|
61
|
-
|
|
62
|
-
headers=
|
|
63
|
-
cookies=
|
|
111
|
+
url=url,
|
|
112
|
+
headers=headers,
|
|
113
|
+
cookies=cookies,
|
|
64
114
|
)
|
|
65
115
|
m_data = json.loads(result.text)
|
|
66
116
|
update_time = m_data['data']['updateTime']
|
|
67
117
|
# pt_data = data['data']['data'][0] # 平台流量
|
|
68
118
|
# gg_data = data['data']['data'][1] # 广告流量
|
|
119
|
+
datas = []
|
|
69
120
|
for all_data in m_data['data']['data']:
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
121
|
+
datas.append(
|
|
122
|
+
{
|
|
123
|
+
'frontPayByrCnt': all_data['frontPayByrCnt']['value'],
|
|
124
|
+
'一级标识id': all_data['pageId']['value'],
|
|
125
|
+
'二级标识id': '',
|
|
126
|
+
'三级标识id': '',
|
|
127
|
+
'一级来源': all_data['pageName']['value'],
|
|
128
|
+
'二级来源': '',
|
|
129
|
+
'三级来源': '',
|
|
130
|
+
'活动商品访客数（定金期）': all_data['frontPreheatUv']['value'],
|
|
131
|
+
'定金支付买家数': all_data['frontPayByrCnt']['value'],
|
|
132
|
+
'定金支付转化率': all_data['frontPayRate']['value'],
|
|
133
|
+
'日期': all_data['statDateStr']['value'],
|
|
134
|
+
'更新时间': update_time,
|
|
135
|
+
'促销活动': '2024双11预售',
|
|
136
|
+
'版块': '流量来源',
|
|
137
|
+
}
|
|
138
|
+
)
|
|
85
139
|
if 'children' not in all_data.keys(): # 这一句有点多余，因为一级来源必定细分有二级来源
|
|
86
140
|
continue
|
|
87
141
|
for children_data in all_data['children']:
|
|
88
142
|
one_source_id = children_data['pPageId']['value']
|
|
89
143
|
one_source_name = children_data['pPageName']['value']
|
|
90
|
-
|
|
144
|
+
datas.append(
|
|
91
145
|
{
|
|
92
146
|
'frontPayByrCnt': children_data['frontPayByrCnt']['value'],
|
|
93
147
|
'一级标识id': children_data['pPageId']['value'],
|
|
@@ -102,6 +156,7 @@ class RequestData:
|
|
|
102
156
|
'日期': children_data['statDateStr']['value'],
|
|
103
157
|
'更新时间': update_time,
|
|
104
158
|
'促销活动': '2024双11预售',
|
|
159
|
+
'版块': '流量来源',
|
|
105
160
|
}
|
|
106
161
|
)
|
|
107
162
|
# print(children_data['children'])
|
|
@@ -111,7 +166,7 @@ class RequestData:
|
|
|
111
166
|
for children_children_data in children_data['children']:
|
|
112
167
|
# print(children_children_data)
|
|
113
168
|
# print(one_source_name)
|
|
114
|
-
|
|
169
|
+
datas.append(
|
|
115
170
|
{
|
|
116
171
|
'frontPayByrCnt': children_children_data['frontPayByrCnt']['value'],
|
|
117
172
|
'一级标识id': one_source_id,
|
|
@@ -126,14 +181,276 @@ class RequestData:
|
|
|
126
181
|
'日期': children_children_data['statDateStr']['value'],
|
|
127
182
|
'更新时间': update_time,
|
|
128
183
|
'促销活动': '2024双11预售',
|
|
184
|
+
'版块': '流量来源',
|
|
129
185
|
}
|
|
130
186
|
)
|
|
131
|
-
for item in
|
|
187
|
+
for item in datas:
|
|
132
188
|
if item['日期'] != '':
|
|
133
189
|
item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
|
|
134
190
|
if self.is_json_file:
|
|
135
|
-
|
|
136
|
-
|
|
191
|
+
if self.path and self.filename:
|
|
192
|
+
with open(os.path.join(self.path, f'{self.filename}.json'), 'w') as f:
|
|
193
|
+
json.dump(datas, f, ensure_ascii=False, sort_keys=True, indent=4)
|
|
194
|
+
else:
|
|
195
|
+
print(f'尚未指定 self.path/ self.filename')
|
|
196
|
+
df = pd.DataFrame(datas)
|
|
197
|
+
df.fillna('0', inplace=True)
|
|
198
|
+
df = df.astype(
|
|
199
|
+
{
|
|
200
|
+
'frontPayByrCnt': int,
|
|
201
|
+
'一级标识id': str,
|
|
202
|
+
'二级标识id': str,
|
|
203
|
+
'三级标识id': str,
|
|
204
|
+
'一级来源': str,
|
|
205
|
+
'二级来源': str,
|
|
206
|
+
'三级来源': str,
|
|
207
|
+
'活动商品访客数（定金期）': int,
|
|
208
|
+
'定金支付买家数': int,
|
|
209
|
+
'促销活动': str,
|
|
210
|
+
'版块': str,
|
|
211
|
+
}, errors='raise')
|
|
212
|
+
return '活动分析2', '2024双11预售实时流量分析', df # 注意这些是实际数据表名字
|
|
213
|
+
|
|
214
|
+
def qxg_ll(self):
|
|
215
|
+
flow_biz_types = {
|
|
216
|
+
'classic': '非全站推广期',
|
|
217
|
+
'qzt': '全站推广期',
|
|
218
|
+
}
|
|
219
|
+
page_types = {
|
|
220
|
+
'item': '商品流量',
|
|
221
|
+
'shop': '店铺流量',
|
|
222
|
+
'live': '直播流量',
|
|
223
|
+
'content': '内容流量',
|
|
224
|
+
}
|
|
225
|
+
for k_flow, v_flow in flow_biz_types.items():
|
|
226
|
+
for k_page, v_page in page_types.items():
|
|
227
|
+
if v_flow == '全站推广期' and v_page != '商品流量':
|
|
228
|
+
continue # 只有商品流量才可以传 qzt值
|
|
229
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
|
230
|
+
print(f'{now} {v_flow} -> {v_page}: 正在获取数据...')
|
|
231
|
+
date = datetime.date.today().strftime('%Y%m%d')
|
|
232
|
+
url = (
|
|
233
|
+
f'https://sycm.taobao.com/flow/v5/live/shop/source/tree/v4.json?'
|
|
234
|
+
f'dateRange={date}%7C{date}'
|
|
235
|
+
f'&dateType=today'
|
|
236
|
+
f'&order=desc'
|
|
237
|
+
f'&orderBy=uv'
|
|
238
|
+
f'&flowBizType={k_flow}' # classic: 非全站推广期，qzt: 全站推广期（只有商品流量才可以传 qzt值）
|
|
239
|
+
f'&pageType={k_page}' # item：商品流量，shop: 店铺流量，live: 直播流量， content： 内容流量
|
|
240
|
+
f'&crowdType=all'
|
|
241
|
+
f'&activityId=92072444'
|
|
242
|
+
f'&indexCode=uv'
|
|
243
|
+
# f'&_=1729232086296'
|
|
244
|
+
# f'&token=2507b8098'
|
|
245
|
+
)
|
|
246
|
+
headers = {
|
|
247
|
+
# "referer": "https://dmp.taobao.com/index_new.html",
|
|
248
|
+
'User-Agent': ua_sj.get_ua(),
|
|
249
|
+
}
|
|
250
|
+
cookies = {
|
|
251
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
|
252
|
+
self.qxg_ll_data(
|
|
253
|
+
url=url,
|
|
254
|
+
headers=headers,
|
|
255
|
+
cookies=cookies,
|
|
256
|
+
flow_biz_type=v_flow,
|
|
257
|
+
page_type=v_page,
|
|
258
|
+
)
|
|
259
|
+
time.sleep(random.randint(5, 10))
|
|
260
|
+
df = pd.concat(self.datas)
|
|
261
|
+
df.fillna(0, inplace=True)
|
|
262
|
+
df = df.astype(
|
|
263
|
+
{
|
|
264
|
+
'支付买家数': int,
|
|
265
|
+
'详情页访客数': int,
|
|
266
|
+
'来源等级': int,
|
|
267
|
+
'showDetailChannel': int,
|
|
268
|
+
}, errors='raise')
|
|
269
|
+
# df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
|
|
270
|
+
return '活动分析2', '2024双11抢先购预热期流量来源', df # 注意这些是实际数据表名字
|
|
271
|
+
|
|
272
|
+
def qxg_ll_data(self, url, headers, cookies, flow_biz_type, page_type):
|
|
273
|
+
""" 抢先购 流量来源 """
|
|
274
|
+
result = requests.get(
|
|
275
|
+
url=url,
|
|
276
|
+
headers=headers,
|
|
277
|
+
cookies=cookies,
|
|
278
|
+
)
|
|
279
|
+
json_datas = json.loads(result.text)
|
|
280
|
+
update_time = json_datas['data']['updateTime']
|
|
281
|
+
# print(update_time)
|
|
282
|
+
datas = []
|
|
283
|
+
json_datas = json_datas['data']['data']
|
|
284
|
+
dict_data = {}
|
|
285
|
+
if page_type == '直播流量' or page_type == '内容流量':
|
|
286
|
+
for item in json_datas:
|
|
287
|
+
datas.append(
|
|
288
|
+
{
|
|
289
|
+
'访客数': item['uv']['value'],
|
|
290
|
+
'pageId': item['pageId']['value'],
|
|
291
|
+
'0级来源': item['pageName']['value'],
|
|
292
|
+
'pPageId': item['pPageId']['value'],
|
|
293
|
+
'日期': update_time,
|
|
294
|
+
'更新时间': update_time,
|
|
295
|
+
'促销活动': '2024双11抢先购',
|
|
296
|
+
'版块': '流量来源',
|
|
297
|
+
'来源分类': flow_biz_type,
|
|
298
|
+
'流量类型': page_type,
|
|
299
|
+
})
|
|
300
|
+
json_datas = json_datas[0]['children']
|
|
301
|
+
|
|
302
|
+
for all_data in json_datas:
|
|
303
|
+
# one_source_id = all_data['pageId']['value']
|
|
304
|
+
one_source_name = all_data['pageName']['value']
|
|
305
|
+
# print(all_data)
|
|
306
|
+
for k_first, v_first in all_data.items():
|
|
307
|
+
# print(k_first, v_first)
|
|
308
|
+
|
|
309
|
+
if k_first == 'children':
|
|
310
|
+
continue
|
|
311
|
+
for k_second, v_second in v_first.items():
|
|
312
|
+
if k_second != 'value':
|
|
313
|
+
dict_data.update({k_second: v_second})
|
|
314
|
+
dict_data.update(
|
|
315
|
+
{
|
|
316
|
+
'guideToShortVideoUv': all_data['guideToShortVideoUv']['value'],
|
|
317
|
+
'hiddenIndexgroup': all_data['hiddenIndexgroup']['value'],
|
|
318
|
+
'访客数': all_data['uv']['value'],
|
|
319
|
+
'访客数占比': all_data['uv']['ratio'],
|
|
320
|
+
'支付买家数': all_data['payByrCnt']['value'],
|
|
321
|
+
'详情页访客数': all_data['ipvUvRelate']['value'],
|
|
322
|
+
'支付转化率': all_data['payRate']['value'],
|
|
323
|
+
'orderByrCnt': all_data['orderByrCnt']['value'],
|
|
324
|
+
'showDesc': all_data['showDesc']['value'],
|
|
325
|
+
'showChannel': all_data['showChannel']['value'],
|
|
326
|
+
'来源等级': all_data['pageLevel']['value'],
|
|
327
|
+
'channelType': all_data['channelType']['value'],
|
|
328
|
+
'orderAmt': all_data['orderAmt']['value'],
|
|
329
|
+
'pageId': all_data['pageId']['value'],
|
|
330
|
+
'pPageId': all_data['pPageId']['value'],
|
|
331
|
+
'payAmt': all_data['payAmt']['value'],
|
|
332
|
+
'一级来源': all_data['pageName']['value'],
|
|
333
|
+
'二级来源': '',
|
|
334
|
+
'三级来源': '',
|
|
335
|
+
'showDetailChannel': all_data['showDetailChannel']['value'],
|
|
336
|
+
'pageDesc': all_data['pageDesc']['value'],
|
|
337
|
+
'payPct': all_data['payPct']['value'],
|
|
338
|
+
'pPageId': all_data['pPageId']['value'],
|
|
339
|
+
'crtRate': all_data['crtRate']['value'],
|
|
340
|
+
'日期': update_time,
|
|
341
|
+
'更新时间': update_time,
|
|
342
|
+
'促销活动': '2024双11抢先购',
|
|
343
|
+
'版块': '流量来源',
|
|
344
|
+
'来源分类': flow_biz_type,
|
|
345
|
+
'流量类型': page_type,
|
|
346
|
+
}
|
|
347
|
+
)
|
|
348
|
+
datas.append(dict_data)
|
|
349
|
+
|
|
350
|
+
if 'children' not in all_data.keys(): # 这一句有点多余，因为一级来源必定细分有二级来源
|
|
351
|
+
continue
|
|
352
|
+
|
|
353
|
+
for children_data in all_data['children']:
|
|
354
|
+
# one_source_id = children_data['pPageId']['value']
|
|
355
|
+
second_source_name = children_data['pageName']['value']
|
|
356
|
+
for k_first, v_first in children_data.items():
|
|
357
|
+
# print(k_first, v_first)
|
|
358
|
+
dict_data = {}
|
|
359
|
+
if k_first == 'children':
|
|
360
|
+
continue
|
|
361
|
+
for k_second, v_second in v_first.items():
|
|
362
|
+
if k_second != 'value':
|
|
363
|
+
dict_data.update({k_second: v_second})
|
|
364
|
+
dict_data.update(
|
|
365
|
+
{
|
|
366
|
+
'guideToShortVideoUv': children_data['guideToShortVideoUv']['value'],
|
|
367
|
+
'hiddenIndexgroup': children_data['hiddenIndexgroup']['value'],
|
|
368
|
+
'访客数': children_data['uv']['value'],
|
|
369
|
+
'访客数占比': children_data['uv']['ratio'],
|
|
370
|
+
'支付买家数': children_data['payByrCnt']['value'],
|
|
371
|
+
'详情页访客数': children_data['ipvUvRelate']['value'],
|
|
372
|
+
'支付转化率': children_data['payRate']['value'],
|
|
373
|
+
'orderByrCnt': children_data['orderByrCnt']['value'],
|
|
374
|
+
'showDesc': children_data['showDesc']['value'],
|
|
375
|
+
'showChannel': children_data['showChannel']['value'],
|
|
376
|
+
'来源等级': children_data['pageLevel']['value'],
|
|
377
|
+
'channelType': children_data['channelType']['value'],
|
|
378
|
+
'orderAmt': children_data['orderAmt']['value'],
|
|
379
|
+
'pageId': children_data['pageId']['value'],
|
|
380
|
+
'pPageId': children_data['pPageId']['value'],
|
|
381
|
+
'payAmt': children_data['payAmt']['value'],
|
|
382
|
+
'一级来源': one_source_name,
|
|
383
|
+
'二级来源': children_data['pageName']['value'],
|
|
384
|
+
'三级来源': '',
|
|
385
|
+
'showDetailChannel': children_data['showDetailChannel']['value'],
|
|
386
|
+
'pageDesc': children_data['pageDesc']['value'],
|
|
387
|
+
'payPct': children_data['payPct']['value'],
|
|
388
|
+
'pPageId': children_data['pPageId']['value'],
|
|
389
|
+
'crtRate': children_data['crtRate']['value'],
|
|
390
|
+
'日期': update_time,
|
|
391
|
+
'更新时间': update_time,
|
|
392
|
+
'促销活动': '2024双11抢先购',
|
|
393
|
+
'版块': '流量来源',
|
|
394
|
+
'来源分类': flow_biz_type,
|
|
395
|
+
'流量类型': page_type,
|
|
396
|
+
}
|
|
397
|
+
)
|
|
398
|
+
datas.append(dict_data)
|
|
399
|
+
# print(children_data['children'])
|
|
400
|
+
# print(children_data)
|
|
401
|
+
if 'children' not in children_data.keys(): # 部分二级来源没有细分的三级来源，因为需要跳过 children 字段
|
|
402
|
+
continue
|
|
403
|
+
for children_children_data in children_data['children']:
|
|
404
|
+
# print(children_children_data)
|
|
405
|
+
# print(one_source_name)
|
|
406
|
+
for k_first, v_first in children_data.items():
|
|
407
|
+
# print(k_first, v_first)
|
|
408
|
+
dict_data = {}
|
|
409
|
+
if k_first == 'children':
|
|
410
|
+
continue
|
|
411
|
+
for k_second, v_second in v_first.items():
|
|
412
|
+
if k_second != 'value':
|
|
413
|
+
dict_data.update({k_second: v_second})
|
|
414
|
+
dict_data.update(
|
|
415
|
+
{
|
|
416
|
+
'guideToShortVideoUv': children_children_data['guideToShortVideoUv']['value'],
|
|
417
|
+
'hiddenIndexgroup': children_children_data['hiddenIndexgroup']['value'],
|
|
418
|
+
'访客数': children_children_data['uv']['value'],
|
|
419
|
+
'访客数占比': children_children_data['uv']['ratio'],
|
|
420
|
+
'支付买家数': children_children_data['payByrCnt']['value'],
|
|
421
|
+
'详情页访客数': children_children_data['ipvUvRelate']['value'],
|
|
422
|
+
'支付转化率': children_children_data['payRate']['value'],
|
|
423
|
+
'orderByrCnt': children_children_data['orderByrCnt']['value'],
|
|
424
|
+
'showDesc': children_children_data['showDesc']['value'],
|
|
425
|
+
'showChannel': children_children_data['showChannel']['value'],
|
|
426
|
+
'来源等级': children_children_data['pageLevel']['value'],
|
|
427
|
+
'channelType': children_children_data['channelType']['value'],
|
|
428
|
+
'orderAmt': children_children_data['orderAmt']['value'],
|
|
429
|
+
'pageId': children_children_data['pageId']['value'],
|
|
430
|
+
'pPageId': children_children_data['pPageId']['value'],
|
|
431
|
+
'payAmt': children_children_data['payAmt']['value'],
|
|
432
|
+
'一级来源': one_source_name,
|
|
433
|
+
'二级来源': second_source_name,
|
|
434
|
+
'三级来源': children_children_data['pageName']['value'],
|
|
435
|
+
'showDetailChannel': children_children_data['showDetailChannel']['value'],
|
|
436
|
+
'pageDesc': children_children_data['pageDesc']['value'],
|
|
437
|
+
'payPct': children_children_data['payPct']['value'],
|
|
438
|
+
'pPageId': children_children_data['pPageId']['value'],
|
|
439
|
+
'crtRate': children_children_data['crtRate']['value'],
|
|
440
|
+
'日期': update_time,
|
|
441
|
+
'更新时间': update_time,
|
|
442
|
+
'促销活动': '2024双11抢先购',
|
|
443
|
+
'版块': '流量来源',
|
|
444
|
+
'来源分类': flow_biz_type,
|
|
445
|
+
'流量类型': page_type,
|
|
446
|
+
}
|
|
447
|
+
)
|
|
448
|
+
datas.append(dict_data)
|
|
449
|
+
# for item in datas:
|
|
450
|
+
# if item['日期'] != '':
|
|
451
|
+
# item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
|
|
452
|
+
df = pd.DataFrame(datas)
|
|
453
|
+
self.datas.append(df)
|
|
137
454
|
|
|
138
455
|
def hd_sp(self, date, url, headers, cookies, path, filename, pages=5):
|
|
139
456
|
""" 活动预售页面 分商品效果 """
|
|
@@ -203,82 +520,6 @@ class RequestData:
|
|
|
203
520
|
print(result.text)
|
|
204
521
|
|
|
205
522
|
|
|
206
|
-
def tb_data(service_databases=[], db_name=None, table_name=None):
|
|
207
|
-
""" 2024双11预售实时流量分析 """
|
|
208
|
-
date = datetime.date.today().strftime('%Y%m%d')
|
|
209
|
-
url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
|
|
210
|
-
f'dateRange={date}%7C{date}'
|
|
211
|
-
f'&dateType=today'
|
|
212
|
-
f'&pageSize=10'
|
|
213
|
-
f'&page=1'
|
|
214
|
-
f'&order=desc'
|
|
215
|
-
f'&orderBy=frontPreheatUv' # 必传参数
|
|
216
|
-
f'&activityId=94040472' # 关键，必传参数
|
|
217
|
-
# f'&activityStatus=3'
|
|
218
|
-
# f'&device=2'
|
|
219
|
-
# f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
|
|
220
|
-
# f'&_=1729079731795'
|
|
221
|
-
# f'&token=7e94ba030'
|
|
222
|
-
)
|
|
223
|
-
headers = {
|
|
224
|
-
"referer": "https://dmp.taobao.com/index_new.html",
|
|
225
|
-
'User-Agent': ua_sj.get_ua(),
|
|
226
|
-
}
|
|
227
|
-
cookies = {
|
|
228
|
-
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
|
229
|
-
path = '/Users/xigua/Downloads'
|
|
230
|
-
filename = 'test'
|
|
231
|
-
r = RequestData()
|
|
232
|
-
r.is_json_file = False
|
|
233
|
-
r.request_data(
|
|
234
|
-
date=date,
|
|
235
|
-
url=url,
|
|
236
|
-
headers=headers,
|
|
237
|
-
cookies=cookies,
|
|
238
|
-
path=path,
|
|
239
|
-
filename=filename,
|
|
240
|
-
)
|
|
241
|
-
# print(r.datas)
|
|
242
|
-
df = pd.DataFrame(r.datas)
|
|
243
|
-
# df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
|
|
244
|
-
|
|
245
|
-
if not service_databases:
|
|
246
|
-
return
|
|
247
|
-
if not db_name or not table_name:
|
|
248
|
-
print(f'尚未指定 db_name/table_name 参数')
|
|
249
|
-
return
|
|
250
|
-
for dt in service_databases:
|
|
251
|
-
for service_name, database in dt.items():
|
|
252
|
-
username, password, host, port = get_myconf.select_config_values(
|
|
253
|
-
target_service=service_name,
|
|
254
|
-
database=database,
|
|
255
|
-
)
|
|
256
|
-
m = mysql.MysqlUpload(
|
|
257
|
-
username=username,
|
|
258
|
-
password=password,
|
|
259
|
-
host=host,
|
|
260
|
-
port=port,
|
|
261
|
-
)
|
|
262
|
-
m.df_to_mysql(
|
|
263
|
-
df=df,
|
|
264
|
-
db_name=db_name,
|
|
265
|
-
table_name=table_name,
|
|
266
|
-
move_insert=False, # 先删除，再插入
|
|
267
|
-
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
|
268
|
-
drop_duplicates=False, # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
|
|
269
|
-
filename=None, # 用来追踪处理进度
|
|
270
|
-
service_database=dt, # 字典
|
|
271
|
-
)
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
def company_run():
|
|
275
|
-
if platform.system() == 'Windows' and socket.gethostname() == 'company':
|
|
276
|
-
while True:
|
|
277
|
-
tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2',
|
|
278
|
-
table_name='2024双11预售实时流量分析')
|
|
279
|
-
time.sleep(random.randint(1500, 2000))
|
|
280
|
-
|
|
281
|
-
|
|
282
523
|
def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
|
|
283
524
|
""" 2024双11预售 分商品效果 """
|
|
284
525
|
date = datetime.date.today().strftime('%Y%m%d')
|
|
@@ -320,11 +561,61 @@ def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
|
|
|
320
561
|
df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
|
|
321
562
|
|
|
322
563
|
|
|
564
|
+
def company_run(service_databases=[]):
|
|
565
|
+
# if platform.system() != 'Windows':
|
|
566
|
+
# return
|
|
567
|
+
# if socket.gethostname() != 'company':
|
|
568
|
+
# return
|
|
569
|
+
while True:
|
|
570
|
+
r = RequestData()
|
|
571
|
+
r.is_json_file = False
|
|
572
|
+
|
|
573
|
+
my_data_list = [
|
|
574
|
+
r.ys_ll_data(), # 双 11预售实时流量分析
|
|
575
|
+
r.qxg_hx_data(), # 抢先购 预热期核心指标
|
|
576
|
+
r.qxg_ll() # 抢先购 流量来源
|
|
577
|
+
]
|
|
578
|
+
|
|
579
|
+
results = []
|
|
580
|
+
for my_data in my_data_list:
|
|
581
|
+
db_name, table_name, df = my_data
|
|
582
|
+
if len(df) == 0:
|
|
583
|
+
print(f'{db_name} -> {table_name} has no data')
|
|
584
|
+
continue
|
|
585
|
+
# print(df)
|
|
586
|
+
results.append([db_name, table_name, df])
|
|
587
|
+
|
|
588
|
+
if not service_databases:
|
|
589
|
+
return
|
|
590
|
+
for dt in service_databases:
|
|
591
|
+
for service_name, database in dt.items():
|
|
592
|
+
username, password, host, port = get_myconf.select_config_values(
|
|
593
|
+
target_service=service_name,
|
|
594
|
+
database=database,
|
|
595
|
+
)
|
|
596
|
+
m = mysql.MysqlUpload(
|
|
597
|
+
username=username,
|
|
598
|
+
password=password,
|
|
599
|
+
host=host,
|
|
600
|
+
port=port,
|
|
601
|
+
)
|
|
602
|
+
for result in results:
|
|
603
|
+
db_name, table_name, df = result
|
|
604
|
+
m.df_to_mysql(
|
|
605
|
+
df=df,
|
|
606
|
+
db_name=db_name,
|
|
607
|
+
table_name=table_name,
|
|
608
|
+
move_insert=False, # 先删除，再插入
|
|
609
|
+
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
|
610
|
+
drop_duplicates=False, # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
|
|
611
|
+
filename=None, # 用来追踪处理进度
|
|
612
|
+
service_database=dt, # 字典
|
|
613
|
+
)
|
|
614
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
|
615
|
+
print(f'{now} {db_name} -> {table_name}: 已入库')
|
|
616
|
+
|
|
617
|
+
time.sleep(random.randint(1500, 2000))
|
|
618
|
+
|
|
619
|
+
|
|
323
620
|
if __name__ == '__main__':
|
|
324
|
-
company_run()
|
|
325
|
-
# tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2', table_name='2024双11预售实时流量分析')
|
|
326
|
-
hd_sp_data(
|
|
327
|
-
service_databases=[{'company': 'mysql'}],
|
|
328
|
-
# db_name='生意参谋2',
|
|
329
|
-
# table_name='2024双11预售实时流量分析',
|
|
330
|
-
)
|
|
621
|
+
company_run(service_databases=[{'company': 'mysql'}])
|
|
@@ -38,10 +38,10 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
|
|
|
38
38
|
mdbq/pbix/refresh_all.py,sha256=viOlLCmz9zg61Q2nzjgl8dChfQxnxRd1A_jmQMb2oDM,5918
|
|
39
39
|
mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
|
|
40
40
|
mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
41
|
-
mdbq/req_post/req_tb.py,sha256=
|
|
41
|
+
mdbq/req_post/req_tb.py,sha256=0gVmmVewIpsJVEXkEEW0UeSSPqE9iAyLpMS0P4qjlX0,35270
|
|
42
42
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
|
43
43
|
mdbq/spider/aikucun.py,sha256=fszwS5jml5-S98u5FurBKVS2L4O-0wXFdU6CsGqbWrE,14104
|
|
44
|
-
mdbq-2.3.
|
|
45
|
-
mdbq-2.3.
|
|
46
|
-
mdbq-2.3.
|
|
47
|
-
mdbq-2.3.
|
|
44
|
+
mdbq-2.3.6.dist-info/METADATA,sha256=wz2ymOmXc17yN6AtYhHOLaiXhO2H5vx14ouOrvKA2dQ,245
|
|
45
|
+
mdbq-2.3.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
46
|
+
mdbq-2.3.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
|
47
|
+
mdbq-2.3.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|