mdbq 2.3.4__py3-none-any.whl → 2.3.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/aggregation.py +1 -1
- mdbq/clean/data_clean.py +3 -3
- mdbq/req_post/req_tb.py +425 -134
- mdbq/spider/aikucun.py +18 -16
- {mdbq-2.3.4.dist-info → mdbq-2.3.6.dist-info}/METADATA +1 -1
- {mdbq-2.3.4.dist-info → mdbq-2.3.6.dist-info}/RECORD +8 -8
- {mdbq-2.3.4.dist-info → mdbq-2.3.6.dist-info}/WHEEL +0 -0
- {mdbq-2.3.4.dist-info → mdbq-2.3.6.dist-info}/top_level.txt +0 -0
mdbq/aggregation/aggregation.py
CHANGED
@@ -526,7 +526,7 @@ class DatabaseUpdate:
|
|
526
526
|
elif name.endswith('.csv') and '竞店分析-来源分析-入店搜索词' in name:
|
527
527
|
df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
|
528
528
|
check_remove_file = True
|
529
|
-
elif name.endswith('.csv') and '爱库存_商品榜单' in name:
|
529
|
+
elif name.endswith('.csv') and '爱库存_商品榜单_spu_' in name:
|
530
530
|
df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
|
531
531
|
check_remove_file = True
|
532
532
|
# ----------------------- 京东数据处理分界线 -----------------------
|
mdbq/clean/data_clean.py
CHANGED
@@ -1147,7 +1147,7 @@ class DataClean:
|
|
1147
1147
|
path = self.path
|
1148
1148
|
for root, dirs, files in os.walk(path, topdown=False):
|
1149
1149
|
for name in files:
|
1150
|
-
print(name)
|
1150
|
+
# print(name)
|
1151
1151
|
is_continue = False
|
1152
1152
|
if is_except:
|
1153
1153
|
for item in is_except:
|
@@ -1158,7 +1158,7 @@ class DataClean:
|
|
1158
1158
|
break
|
1159
1159
|
if is_continue: # 需要排除不做处理的文件或文件夹
|
1160
1160
|
continue
|
1161
|
-
print(is_except, is_continue)
|
1161
|
+
# print(is_except, is_continue)
|
1162
1162
|
def bib(paths, _as_month=None):
|
1163
1163
|
"""闭包函数"""
|
1164
1164
|
self.move_files(path=path, _name=name, target_path=paths, _as_month=_as_month)
|
@@ -1348,7 +1348,7 @@ class DataClean:
|
|
1348
1348
|
elif name.endswith('.csv') and '京东推广关键词点击成交报表' in name:
|
1349
1349
|
t_path = str(pathlib.Path(self.source_path, '京东报表/JD推广_关键词报表'))
|
1350
1350
|
bib(t_path, _as_month=True)
|
1351
|
-
elif name.endswith('.csv') and '爱库存
|
1351
|
+
elif name.endswith('.csv') and '爱库存_商品榜单_spu_' in name:
|
1352
1352
|
t_path = str(pathlib.Path(self.source_path, '爱库存/商品榜单'))
|
1353
1353
|
bib(t_path, _as_month=True)
|
1354
1354
|
# 京东分界线 ------- 结束标记
|
mdbq/req_post/req_tb.py
CHANGED
@@ -26,68 +26,122 @@ class RequestData:
|
|
26
26
|
self.path = None
|
27
27
|
self.filename = None
|
28
28
|
self.is_json_file = False
|
29
|
+
self.df = pd.DataFrame()
|
29
30
|
|
30
|
-
def
|
31
|
+
def qxg_hx_data(self):
|
32
|
+
""" 抢先购 预热期核心指标 """
|
33
|
+
date = datetime.date.today().strftime('%Y-%m-%d')
|
34
|
+
url = (f'https://sycm.taobao.com/datawar/v4/activity/detail/kpi/coreIndex/live.json?'
|
35
|
+
f'activityId=92072444'
|
36
|
+
f'&status=1'
|
37
|
+
f'&dateType=today'
|
38
|
+
f'&dateRange={date}%7C{date}'
|
39
|
+
f'&_=1729216673692'
|
40
|
+
f'&token=0939158d0'
|
41
|
+
)
|
42
|
+
headers = {'User-Agent': ua_sj.get_ua()}
|
43
|
+
cookies = {
|
44
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
45
|
+
# cookies = {}
|
46
|
+
path = '/Users/xigua/Downloads'
|
47
|
+
filename = 'test'
|
48
|
+
result = requests.get(
|
49
|
+
url=url,
|
50
|
+
headers=headers,
|
51
|
+
cookies=cookies,
|
52
|
+
)
|
53
|
+
m_data = json.loads(result.text)
|
54
|
+
# print(m_data)
|
55
|
+
update_time = m_data['data']['updateTime']
|
56
|
+
all_data = m_data['data']['data']
|
57
|
+
timestamp = all_data['statDate']['value'] // 1000 # 毫秒转为秒,不然无法转换时间戳
|
58
|
+
|
59
|
+
datas=[{
|
60
|
+
'日期': time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(timestamp))),
|
61
|
+
'预热加购人数': all_data['preheatCartByrCnt']['value'],
|
62
|
+
'加购转化率': all_data['cartRate']['value'],
|
63
|
+
'预热加购件数': all_data['preheatCartItmCnt']['value'],
|
64
|
+
'预热访客数': all_data['preheatUv']['value'],
|
65
|
+
'收藏转化率': all_data['cltRate']['value'],
|
66
|
+
'预热收藏次数': all_data['preheatCltItmCnt']['value'],
|
67
|
+
'预热收藏人数': all_data['preheatCltByrCnt']['value'],
|
68
|
+
'更新时间': update_time,
|
69
|
+
'促销活动': '2024双11抢先购',
|
70
|
+
'版块': '预热期核心指标',
|
71
|
+
}]
|
72
|
+
df = pd.DataFrame(datas)
|
73
|
+
df = df.astype({
|
74
|
+
'预热加购人数': int,
|
75
|
+
'预热加购件数': int,
|
76
|
+
'预热访客数': int,
|
77
|
+
'预热收藏次数': int,
|
78
|
+
'预热收藏人数': int,
|
79
|
+
'促销活动': str,
|
80
|
+
'版块': str,
|
81
|
+
}, errors='raise')
|
82
|
+
return '活动分析2', '2024双11抢先购预热期核心指标', df # 注意这些是实际数据表名字
|
83
|
+
|
84
|
+
def ys_ll_data(self):
|
31
85
|
""" 活动预售页面 流量来源 """
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
86
|
+
date = datetime.date.today().strftime('%Y%m%d')
|
87
|
+
url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
|
88
|
+
f'dateRange={date}%7C{date}'
|
89
|
+
f'&dateType=today'
|
90
|
+
f'&pageSize=10'
|
91
|
+
f'&page=1'
|
92
|
+
f'&order=desc'
|
93
|
+
f'&orderBy=frontPreheatUv' # 必传参数
|
94
|
+
f'&activityId=94040472' # 关键,必传参数
|
95
|
+
# f'&activityStatus=3'
|
96
|
+
# f'&device=2'
|
97
|
+
# f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
|
98
|
+
# f'&_=1729079731795'
|
99
|
+
# f'&token=7e94ba030'
|
100
|
+
)
|
101
|
+
headers = {
|
102
|
+
# "referer": "https://dmp.taobao.com/index_new.html",
|
103
|
+
'User-Agent': ua_sj.get_ua(),
|
104
|
+
}
|
105
|
+
cookies = {
|
106
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
107
|
+
path = '/Users/xigua/Downloads'
|
108
|
+
filename = 'test'
|
53
109
|
|
54
|
-
self.date = date
|
55
|
-
self.url = url
|
56
|
-
self.headers = headers
|
57
|
-
self.cookies = cookies
|
58
|
-
self.path = path
|
59
|
-
self.filename = filename
|
60
110
|
result = requests.get(
|
61
|
-
|
62
|
-
headers=
|
63
|
-
cookies=
|
111
|
+
url=url,
|
112
|
+
headers=headers,
|
113
|
+
cookies=cookies,
|
64
114
|
)
|
65
115
|
m_data = json.loads(result.text)
|
66
116
|
update_time = m_data['data']['updateTime']
|
67
117
|
# pt_data = data['data']['data'][0] # 平台流量
|
68
118
|
# gg_data = data['data']['data'][1] # 广告流量
|
119
|
+
datas = []
|
69
120
|
for all_data in m_data['data']['data']:
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
121
|
+
datas.append(
|
122
|
+
{
|
123
|
+
'frontPayByrCnt': all_data['frontPayByrCnt']['value'],
|
124
|
+
'一级标识id': all_data['pageId']['value'],
|
125
|
+
'二级标识id': '',
|
126
|
+
'三级标识id': '',
|
127
|
+
'一级来源': all_data['pageName']['value'],
|
128
|
+
'二级来源': '',
|
129
|
+
'三级来源': '',
|
130
|
+
'活动商品访客数(定金期)': all_data['frontPreheatUv']['value'],
|
131
|
+
'定金支付买家数': all_data['frontPayByrCnt']['value'],
|
132
|
+
'定金支付转化率': all_data['frontPayRate']['value'],
|
133
|
+
'日期': all_data['statDateStr']['value'],
|
134
|
+
'更新时间': update_time,
|
135
|
+
'促销活动': '2024双11预售',
|
136
|
+
'版块': '流量来源',
|
137
|
+
}
|
138
|
+
)
|
85
139
|
if 'children' not in all_data.keys(): # 这一句有点多余,因为一级来源必定细分有二级来源
|
86
140
|
continue
|
87
141
|
for children_data in all_data['children']:
|
88
142
|
one_source_id = children_data['pPageId']['value']
|
89
143
|
one_source_name = children_data['pPageName']['value']
|
90
|
-
|
144
|
+
datas.append(
|
91
145
|
{
|
92
146
|
'frontPayByrCnt': children_data['frontPayByrCnt']['value'],
|
93
147
|
'一级标识id': children_data['pPageId']['value'],
|
@@ -102,6 +156,7 @@ class RequestData:
|
|
102
156
|
'日期': children_data['statDateStr']['value'],
|
103
157
|
'更新时间': update_time,
|
104
158
|
'促销活动': '2024双11预售',
|
159
|
+
'版块': '流量来源',
|
105
160
|
}
|
106
161
|
)
|
107
162
|
# print(children_data['children'])
|
@@ -111,7 +166,7 @@ class RequestData:
|
|
111
166
|
for children_children_data in children_data['children']:
|
112
167
|
# print(children_children_data)
|
113
168
|
# print(one_source_name)
|
114
|
-
|
169
|
+
datas.append(
|
115
170
|
{
|
116
171
|
'frontPayByrCnt': children_children_data['frontPayByrCnt']['value'],
|
117
172
|
'一级标识id': one_source_id,
|
@@ -126,14 +181,276 @@ class RequestData:
|
|
126
181
|
'日期': children_children_data['statDateStr']['value'],
|
127
182
|
'更新时间': update_time,
|
128
183
|
'促销活动': '2024双11预售',
|
184
|
+
'版块': '流量来源',
|
129
185
|
}
|
130
186
|
)
|
131
|
-
for item in
|
187
|
+
for item in datas:
|
132
188
|
if item['日期'] != '':
|
133
189
|
item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
|
134
190
|
if self.is_json_file:
|
135
|
-
|
136
|
-
|
191
|
+
if self.path and self.filename:
|
192
|
+
with open(os.path.join(self.path, f'{self.filename}.json'), 'w') as f:
|
193
|
+
json.dump(datas, f, ensure_ascii=False, sort_keys=True, indent=4)
|
194
|
+
else:
|
195
|
+
print(f'尚未指定 self.path/ self.filename')
|
196
|
+
df = pd.DataFrame(datas)
|
197
|
+
df.fillna('0', inplace=True)
|
198
|
+
df = df.astype(
|
199
|
+
{
|
200
|
+
'frontPayByrCnt': int,
|
201
|
+
'一级标识id': str,
|
202
|
+
'二级标识id': str,
|
203
|
+
'三级标识id': str,
|
204
|
+
'一级来源': str,
|
205
|
+
'二级来源': str,
|
206
|
+
'三级来源': str,
|
207
|
+
'活动商品访客数(定金期)': int,
|
208
|
+
'定金支付买家数': int,
|
209
|
+
'促销活动': str,
|
210
|
+
'版块': str,
|
211
|
+
}, errors='raise')
|
212
|
+
return '活动分析2', '2024双11预售实时流量分析', df # 注意这些是实际数据表名字
|
213
|
+
|
214
|
+
def qxg_ll(self):
|
215
|
+
flow_biz_types = {
|
216
|
+
'classic': '非全站推广期',
|
217
|
+
'qzt': '全站推广期',
|
218
|
+
}
|
219
|
+
page_types = {
|
220
|
+
'item': '商品流量',
|
221
|
+
'shop': '店铺流量',
|
222
|
+
'live': '直播流量',
|
223
|
+
'content': '内容流量',
|
224
|
+
}
|
225
|
+
for k_flow, v_flow in flow_biz_types.items():
|
226
|
+
for k_page, v_page in page_types.items():
|
227
|
+
if v_flow == '全站推广期' and v_page != '商品流量':
|
228
|
+
continue # 只有商品流量才可以传 qzt值
|
229
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
230
|
+
print(f'{now} {v_flow} -> {v_page}: 正在获取数据...')
|
231
|
+
date = datetime.date.today().strftime('%Y%m%d')
|
232
|
+
url = (
|
233
|
+
f'https://sycm.taobao.com/flow/v5/live/shop/source/tree/v4.json?'
|
234
|
+
f'dateRange={date}%7C{date}'
|
235
|
+
f'&dateType=today'
|
236
|
+
f'&order=desc'
|
237
|
+
f'&orderBy=uv'
|
238
|
+
f'&flowBizType={k_flow}' # classic: 非全站推广期,qzt: 全站推广期(只有商品流量才可以传 qzt值)
|
239
|
+
f'&pageType={k_page}' # item:商品流量,shop: 店铺流量,live: 直播流量, content: 内容流量
|
240
|
+
f'&crowdType=all'
|
241
|
+
f'&activityId=92072444'
|
242
|
+
f'&indexCode=uv'
|
243
|
+
# f'&_=1729232086296'
|
244
|
+
# f'&token=2507b8098'
|
245
|
+
)
|
246
|
+
headers = {
|
247
|
+
# "referer": "https://dmp.taobao.com/index_new.html",
|
248
|
+
'User-Agent': ua_sj.get_ua(),
|
249
|
+
}
|
250
|
+
cookies = {
|
251
|
+
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
252
|
+
self.qxg_ll_data(
|
253
|
+
url=url,
|
254
|
+
headers=headers,
|
255
|
+
cookies=cookies,
|
256
|
+
flow_biz_type=v_flow,
|
257
|
+
page_type=v_page,
|
258
|
+
)
|
259
|
+
time.sleep(random.randint(5, 10))
|
260
|
+
df = pd.concat(self.datas)
|
261
|
+
df.fillna(0, inplace=True)
|
262
|
+
df = df.astype(
|
263
|
+
{
|
264
|
+
'支付买家数': int,
|
265
|
+
'详情页访客数': int,
|
266
|
+
'来源等级': int,
|
267
|
+
'showDetailChannel': int,
|
268
|
+
}, errors='raise')
|
269
|
+
# df.to_csv('/Users/xigua/Downloads/test.csv', index=False, header=True, encoding='utf-8_sig')
|
270
|
+
return '活动分析2', '2024双11抢先购预热期流量来源', df # 注意这些是实际数据表名字
|
271
|
+
|
272
|
+
def qxg_ll_data(self, url, headers, cookies, flow_biz_type, page_type):
|
273
|
+
""" 抢先购 流量来源 """
|
274
|
+
result = requests.get(
|
275
|
+
url=url,
|
276
|
+
headers=headers,
|
277
|
+
cookies=cookies,
|
278
|
+
)
|
279
|
+
json_datas = json.loads(result.text)
|
280
|
+
update_time = json_datas['data']['updateTime']
|
281
|
+
# print(update_time)
|
282
|
+
datas = []
|
283
|
+
json_datas = json_datas['data']['data']
|
284
|
+
dict_data = {}
|
285
|
+
if page_type == '直播流量' or page_type == '内容流量':
|
286
|
+
for item in json_datas:
|
287
|
+
datas.append(
|
288
|
+
{
|
289
|
+
'访客数': item['uv']['value'],
|
290
|
+
'pageId': item['pageId']['value'],
|
291
|
+
'0级来源': item['pageName']['value'],
|
292
|
+
'pPageId': item['pPageId']['value'],
|
293
|
+
'日期': update_time,
|
294
|
+
'更新时间': update_time,
|
295
|
+
'促销活动': '2024双11抢先购',
|
296
|
+
'版块': '流量来源',
|
297
|
+
'来源分类': flow_biz_type,
|
298
|
+
'流量类型': page_type,
|
299
|
+
})
|
300
|
+
json_datas = json_datas[0]['children']
|
301
|
+
|
302
|
+
for all_data in json_datas:
|
303
|
+
# one_source_id = all_data['pageId']['value']
|
304
|
+
one_source_name = all_data['pageName']['value']
|
305
|
+
# print(all_data)
|
306
|
+
for k_first, v_first in all_data.items():
|
307
|
+
# print(k_first, v_first)
|
308
|
+
|
309
|
+
if k_first == 'children':
|
310
|
+
continue
|
311
|
+
for k_second, v_second in v_first.items():
|
312
|
+
if k_second != 'value':
|
313
|
+
dict_data.update({k_second: v_second})
|
314
|
+
dict_data.update(
|
315
|
+
{
|
316
|
+
'guideToShortVideoUv': all_data['guideToShortVideoUv']['value'],
|
317
|
+
'hiddenIndexgroup': all_data['hiddenIndexgroup']['value'],
|
318
|
+
'访客数': all_data['uv']['value'],
|
319
|
+
'访客数占比': all_data['uv']['ratio'],
|
320
|
+
'支付买家数': all_data['payByrCnt']['value'],
|
321
|
+
'详情页访客数': all_data['ipvUvRelate']['value'],
|
322
|
+
'支付转化率': all_data['payRate']['value'],
|
323
|
+
'orderByrCnt': all_data['orderByrCnt']['value'],
|
324
|
+
'showDesc': all_data['showDesc']['value'],
|
325
|
+
'showChannel': all_data['showChannel']['value'],
|
326
|
+
'来源等级': all_data['pageLevel']['value'],
|
327
|
+
'channelType': all_data['channelType']['value'],
|
328
|
+
'orderAmt': all_data['orderAmt']['value'],
|
329
|
+
'pageId': all_data['pageId']['value'],
|
330
|
+
'pPageId': all_data['pPageId']['value'],
|
331
|
+
'payAmt': all_data['payAmt']['value'],
|
332
|
+
'一级来源': all_data['pageName']['value'],
|
333
|
+
'二级来源': '',
|
334
|
+
'三级来源': '',
|
335
|
+
'showDetailChannel': all_data['showDetailChannel']['value'],
|
336
|
+
'pageDesc': all_data['pageDesc']['value'],
|
337
|
+
'payPct': all_data['payPct']['value'],
|
338
|
+
'pPageId': all_data['pPageId']['value'],
|
339
|
+
'crtRate': all_data['crtRate']['value'],
|
340
|
+
'日期': update_time,
|
341
|
+
'更新时间': update_time,
|
342
|
+
'促销活动': '2024双11抢先购',
|
343
|
+
'版块': '流量来源',
|
344
|
+
'来源分类': flow_biz_type,
|
345
|
+
'流量类型': page_type,
|
346
|
+
}
|
347
|
+
)
|
348
|
+
datas.append(dict_data)
|
349
|
+
|
350
|
+
if 'children' not in all_data.keys(): # 这一句有点多余,因为一级来源必定细分有二级来源
|
351
|
+
continue
|
352
|
+
|
353
|
+
for children_data in all_data['children']:
|
354
|
+
# one_source_id = children_data['pPageId']['value']
|
355
|
+
second_source_name = children_data['pageName']['value']
|
356
|
+
for k_first, v_first in children_data.items():
|
357
|
+
# print(k_first, v_first)
|
358
|
+
dict_data = {}
|
359
|
+
if k_first == 'children':
|
360
|
+
continue
|
361
|
+
for k_second, v_second in v_first.items():
|
362
|
+
if k_second != 'value':
|
363
|
+
dict_data.update({k_second: v_second})
|
364
|
+
dict_data.update(
|
365
|
+
{
|
366
|
+
'guideToShortVideoUv': children_data['guideToShortVideoUv']['value'],
|
367
|
+
'hiddenIndexgroup': children_data['hiddenIndexgroup']['value'],
|
368
|
+
'访客数': children_data['uv']['value'],
|
369
|
+
'访客数占比': children_data['uv']['ratio'],
|
370
|
+
'支付买家数': children_data['payByrCnt']['value'],
|
371
|
+
'详情页访客数': children_data['ipvUvRelate']['value'],
|
372
|
+
'支付转化率': children_data['payRate']['value'],
|
373
|
+
'orderByrCnt': children_data['orderByrCnt']['value'],
|
374
|
+
'showDesc': children_data['showDesc']['value'],
|
375
|
+
'showChannel': children_data['showChannel']['value'],
|
376
|
+
'来源等级': children_data['pageLevel']['value'],
|
377
|
+
'channelType': children_data['channelType']['value'],
|
378
|
+
'orderAmt': children_data['orderAmt']['value'],
|
379
|
+
'pageId': children_data['pageId']['value'],
|
380
|
+
'pPageId': children_data['pPageId']['value'],
|
381
|
+
'payAmt': children_data['payAmt']['value'],
|
382
|
+
'一级来源': one_source_name,
|
383
|
+
'二级来源': children_data['pageName']['value'],
|
384
|
+
'三级来源': '',
|
385
|
+
'showDetailChannel': children_data['showDetailChannel']['value'],
|
386
|
+
'pageDesc': children_data['pageDesc']['value'],
|
387
|
+
'payPct': children_data['payPct']['value'],
|
388
|
+
'pPageId': children_data['pPageId']['value'],
|
389
|
+
'crtRate': children_data['crtRate']['value'],
|
390
|
+
'日期': update_time,
|
391
|
+
'更新时间': update_time,
|
392
|
+
'促销活动': '2024双11抢先购',
|
393
|
+
'版块': '流量来源',
|
394
|
+
'来源分类': flow_biz_type,
|
395
|
+
'流量类型': page_type,
|
396
|
+
}
|
397
|
+
)
|
398
|
+
datas.append(dict_data)
|
399
|
+
# print(children_data['children'])
|
400
|
+
# print(children_data)
|
401
|
+
if 'children' not in children_data.keys(): # 部分二级来源没有细分的三级来源,因为需要跳过 children 字段
|
402
|
+
continue
|
403
|
+
for children_children_data in children_data['children']:
|
404
|
+
# print(children_children_data)
|
405
|
+
# print(one_source_name)
|
406
|
+
for k_first, v_first in children_data.items():
|
407
|
+
# print(k_first, v_first)
|
408
|
+
dict_data = {}
|
409
|
+
if k_first == 'children':
|
410
|
+
continue
|
411
|
+
for k_second, v_second in v_first.items():
|
412
|
+
if k_second != 'value':
|
413
|
+
dict_data.update({k_second: v_second})
|
414
|
+
dict_data.update(
|
415
|
+
{
|
416
|
+
'guideToShortVideoUv': children_children_data['guideToShortVideoUv']['value'],
|
417
|
+
'hiddenIndexgroup': children_children_data['hiddenIndexgroup']['value'],
|
418
|
+
'访客数': children_children_data['uv']['value'],
|
419
|
+
'访客数占比': children_children_data['uv']['ratio'],
|
420
|
+
'支付买家数': children_children_data['payByrCnt']['value'],
|
421
|
+
'详情页访客数': children_children_data['ipvUvRelate']['value'],
|
422
|
+
'支付转化率': children_children_data['payRate']['value'],
|
423
|
+
'orderByrCnt': children_children_data['orderByrCnt']['value'],
|
424
|
+
'showDesc': children_children_data['showDesc']['value'],
|
425
|
+
'showChannel': children_children_data['showChannel']['value'],
|
426
|
+
'来源等级': children_children_data['pageLevel']['value'],
|
427
|
+
'channelType': children_children_data['channelType']['value'],
|
428
|
+
'orderAmt': children_children_data['orderAmt']['value'],
|
429
|
+
'pageId': children_children_data['pageId']['value'],
|
430
|
+
'pPageId': children_children_data['pPageId']['value'],
|
431
|
+
'payAmt': children_children_data['payAmt']['value'],
|
432
|
+
'一级来源': one_source_name,
|
433
|
+
'二级来源': second_source_name,
|
434
|
+
'三级来源': children_children_data['pageName']['value'],
|
435
|
+
'showDetailChannel': children_children_data['showDetailChannel']['value'],
|
436
|
+
'pageDesc': children_children_data['pageDesc']['value'],
|
437
|
+
'payPct': children_children_data['payPct']['value'],
|
438
|
+
'pPageId': children_children_data['pPageId']['value'],
|
439
|
+
'crtRate': children_children_data['crtRate']['value'],
|
440
|
+
'日期': update_time,
|
441
|
+
'更新时间': update_time,
|
442
|
+
'促销活动': '2024双11抢先购',
|
443
|
+
'版块': '流量来源',
|
444
|
+
'来源分类': flow_biz_type,
|
445
|
+
'流量类型': page_type,
|
446
|
+
}
|
447
|
+
)
|
448
|
+
datas.append(dict_data)
|
449
|
+
# for item in datas:
|
450
|
+
# if item['日期'] != '':
|
451
|
+
# item.update({'日期': f'{item['日期'][0:4]}-{item['日期'][4:6]}-{item['日期'][6:8]}'})
|
452
|
+
df = pd.DataFrame(datas)
|
453
|
+
self.datas.append(df)
|
137
454
|
|
138
455
|
def hd_sp(self, date, url, headers, cookies, path, filename, pages=5):
|
139
456
|
""" 活动预售页面 分商品效果 """
|
@@ -203,82 +520,6 @@ class RequestData:
|
|
203
520
|
print(result.text)
|
204
521
|
|
205
522
|
|
206
|
-
def tb_data(service_databases=[], db_name=None, table_name=None):
|
207
|
-
""" 2024双11预售实时流量分析 """
|
208
|
-
date = datetime.date.today().strftime('%Y%m%d')
|
209
|
-
url = (f'https://sycm.taobao.com/datawar/v6/activity/detail/guide/chl/presale/online/v4.json?'
|
210
|
-
f'dateRange={date}%7C{date}'
|
211
|
-
f'&dateType=today'
|
212
|
-
f'&pageSize=10'
|
213
|
-
f'&page=1'
|
214
|
-
f'&order=desc'
|
215
|
-
f'&orderBy=frontPreheatUv' # 必传参数
|
216
|
-
f'&activityId=94040472' # 关键,必传参数
|
217
|
-
# f'&activityStatus=3'
|
218
|
-
# f'&device=2'
|
219
|
-
# f'&indexCode=frontPreheatUv%2CfrontPayByrCnt%2CfrontPayRate'
|
220
|
-
# f'&_=1729079731795'
|
221
|
-
# f'&token=7e94ba030'
|
222
|
-
)
|
223
|
-
headers = {
|
224
|
-
"referer": "https://dmp.taobao.com/index_new.html",
|
225
|
-
'User-Agent': ua_sj.get_ua(),
|
226
|
-
}
|
227
|
-
cookies = {
|
228
|
-
'session': 't=c198527347800dafa75165f084784668; thw=cn; xlly_s=1; _tb_token_=rPWSGun4nUou9aKxviPg; _samesite_flag_=true; 3PcFlag=1729054801593; cookie2=130befc055eed2df29935197bd2b514b; sgcookie=E100aLOltfWHqLLH1qtyH3it%2BLrGH2v3MAnIBdSfu7xwjEpSyh101lblDVcj3zGpAOLv%2FXcrVNbT%2FN%2BI8KZeCoE4HBzHQk0ANtSqjOG5gIzdKamfirBxGWJyVEccitvvDZhK; unb=2210244713719; sn=%E4%B8%87%E9%87%8C%E9%A9%AC%E5%AE%98%E6%96%B9%E6%97%97%E8%88%B0%E5%BA%97%3A%E6%8E%A8%E5%B9%BF; uc1=cookie21=W5iHLLyFfoaZ&cookie14=UoYcCoAfJ7pSQA%3D%3D; csg=1e2bdb8a; _cc_=Vq8l%2BKCLiw%3D%3D; cancelledSubSites=empty; skt=f813f8478f7318f8; v=0; cna=8+iAHxeojXcCAXjsc5Mt+BAV; mtop_partitioned_detect=1; _m_h5_tk=88c56a84a93c1199f8abe086a132c7eb_1729068459392; _m_h5_tk_enc=4b0ed8316f46edae303547d3863982a4; XSRF-TOKEN=4ef3d151-14c4-445a-9249-595e9a24df75; JSESSIONID=9EE8C8DCF6162DCA2FE0187C29BF0B8A; tfstk=gyaEdSAx842sxMbj1f3rgEWrJ50LN2XbxzMSZ7VoOvDheWNubSerd_IKRlkzIRk3O76JzQqgCk9QZzGuzR3n2kMSdYuzw-51hZ_b9W3--t6flZ3LgJuxZBYHFAYiG40ZtLV_9W3J6C9lclVpUV2YVJ0uEVmiwj0kr00l_ccjZ4YnqexMIAhor4YoqVDiwjvkr80l_5DttHciSWVk7jihGd0FW1QAcqH0tA8kuIhKxg2JVH-emXiZncbekEC-TDk0tAWAnqwo4JoU5wJxTlV4BXyRke3n4kqm-zWV8VVYfJcaEt-rIozLzmaF3nH3JYeq-lWM840Kg7obf_xqCuVT7czFcQhTR74KcqbvKYZ_gzlzyTQa3W2Umm4HLgz6efAQOzEeE3on6fkf_1ySvoccWpB-m3K-jqhZh6GB23nnhfkf_1-J2cDo_x1IO; isg=BLm5J8RI-qdgDKdAgF_DSgcFyCOTxq14BgKdB9vjgONeYsD0IReUSUT05GaUWkWw'}
|
229
|
-
path = '/Users/xigua/Downloads'
|
230
|
-
filename = 'test'
|
231
|
-
r = RequestData()
|
232
|
-
r.is_json_file = False
|
233
|
-
r.request_data(
|
234
|
-
date=date,
|
235
|
-
url=url,
|
236
|
-
headers=headers,
|
237
|
-
cookies=cookies,
|
238
|
-
path=path,
|
239
|
-
filename=filename,
|
240
|
-
)
|
241
|
-
# print(r.datas)
|
242
|
-
df = pd.DataFrame(r.datas)
|
243
|
-
# df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
|
244
|
-
|
245
|
-
if not service_databases:
|
246
|
-
return
|
247
|
-
if not db_name or not table_name:
|
248
|
-
print(f'尚未指定 db_name/table_name 参数')
|
249
|
-
return
|
250
|
-
for dt in service_databases:
|
251
|
-
for service_name, database in dt.items():
|
252
|
-
username, password, host, port = get_myconf.select_config_values(
|
253
|
-
target_service=service_name,
|
254
|
-
database=database,
|
255
|
-
)
|
256
|
-
m = mysql.MysqlUpload(
|
257
|
-
username=username,
|
258
|
-
password=password,
|
259
|
-
host=host,
|
260
|
-
port=port,
|
261
|
-
)
|
262
|
-
m.df_to_mysql(
|
263
|
-
df=df,
|
264
|
-
db_name=db_name,
|
265
|
-
table_name=table_name,
|
266
|
-
move_insert=False, # 先删除,再插入
|
267
|
-
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
268
|
-
drop_duplicates=False, # 值为 True 时检查重复数据再插入,反之直接上传,会比较慢
|
269
|
-
filename=None, # 用来追踪处理进度
|
270
|
-
service_database=dt, # 字典
|
271
|
-
)
|
272
|
-
|
273
|
-
|
274
|
-
def company_run():
|
275
|
-
if platform.system() == 'Windows' and socket.gethostname() == 'company':
|
276
|
-
while True:
|
277
|
-
tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2',
|
278
|
-
table_name='2024双11预售实时流量分析')
|
279
|
-
time.sleep(random.randint(1500, 2000))
|
280
|
-
|
281
|
-
|
282
523
|
def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
|
283
524
|
""" 2024双11预售 分商品效果 """
|
284
525
|
date = datetime.date.today().strftime('%Y%m%d')
|
@@ -320,11 +561,61 @@ def hd_sp_data(service_databases=[], db_name=None, table_name=None, pages=5):
|
|
320
561
|
df.to_csv(os.path.join(path, 'test.csv'), index=False, header=True, encoding='utf-8_sig')
|
321
562
|
|
322
563
|
|
564
|
+
def company_run(service_databases=[]):
|
565
|
+
# if platform.system() != 'Windows':
|
566
|
+
# return
|
567
|
+
# if socket.gethostname() != 'company':
|
568
|
+
# return
|
569
|
+
while True:
|
570
|
+
r = RequestData()
|
571
|
+
r.is_json_file = False
|
572
|
+
|
573
|
+
my_data_list = [
|
574
|
+
r.ys_ll_data(), # 双 11预售实时流量分析
|
575
|
+
r.qxg_hx_data(), # 抢先购 预热期核心指标
|
576
|
+
r.qxg_ll() # 抢先购 流量来源
|
577
|
+
]
|
578
|
+
|
579
|
+
results = []
|
580
|
+
for my_data in my_data_list:
|
581
|
+
db_name, table_name, df = my_data
|
582
|
+
if len(df) == 0:
|
583
|
+
print(f'{db_name} -> {table_name} has no data')
|
584
|
+
continue
|
585
|
+
# print(df)
|
586
|
+
results.append([db_name, table_name, df])
|
587
|
+
|
588
|
+
if not service_databases:
|
589
|
+
return
|
590
|
+
for dt in service_databases:
|
591
|
+
for service_name, database in dt.items():
|
592
|
+
username, password, host, port = get_myconf.select_config_values(
|
593
|
+
target_service=service_name,
|
594
|
+
database=database,
|
595
|
+
)
|
596
|
+
m = mysql.MysqlUpload(
|
597
|
+
username=username,
|
598
|
+
password=password,
|
599
|
+
host=host,
|
600
|
+
port=port,
|
601
|
+
)
|
602
|
+
for result in results:
|
603
|
+
db_name, table_name, df = result
|
604
|
+
m.df_to_mysql(
|
605
|
+
df=df,
|
606
|
+
db_name=db_name,
|
607
|
+
table_name=table_name,
|
608
|
+
move_insert=False, # 先删除,再插入
|
609
|
+
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
610
|
+
drop_duplicates=False, # 值为 True 时检查重复数据再插入,反之直接上传,会比较慢
|
611
|
+
filename=None, # 用来追踪处理进度
|
612
|
+
service_database=dt, # 字典
|
613
|
+
)
|
614
|
+
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
|
615
|
+
print(f'{now} {db_name} -> {table_name}: 已入库')
|
616
|
+
|
617
|
+
time.sleep(random.randint(1500, 2000))
|
618
|
+
|
619
|
+
|
323
620
|
if __name__ == '__main__':
|
324
|
-
company_run()
|
325
|
-
# tb_data(service_databases=[{'company': 'mysql'}], db_name='生意参谋2', table_name='2024双11预售实时流量分析')
|
326
|
-
hd_sp_data(
|
327
|
-
service_databases=[{'company': 'mysql'}],
|
328
|
-
# db_name='生意参谋2',
|
329
|
-
# table_name='2024双11预售实时流量分析',
|
330
|
-
)
|
621
|
+
company_run(service_databases=[{'company': 'mysql'}])
|
mdbq/spider/aikucun.py
CHANGED
@@ -43,11 +43,12 @@ else:
|
|
43
43
|
Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
|
44
44
|
|
45
45
|
|
46
|
-
def
|
46
|
+
def get_cookie_aikucun():
|
47
47
|
"""
|
48
48
|
"""
|
49
49
|
_url = 'https://gray-merc.aikucun.com/index.html'
|
50
|
-
cookie_path = '
|
50
|
+
cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
|
51
|
+
filename_aikucun = 'cookie_aikucun.txt'
|
51
52
|
print(_url)
|
52
53
|
|
53
54
|
option = webdriver.ChromeOptions() # 浏览器启动选项
|
@@ -79,7 +80,6 @@ def test():
|
|
79
80
|
service = Service(chromedriver_path)
|
80
81
|
_driver = webdriver.Chrome(service=service, options=option) # 创建Chrome驱动程序实例
|
81
82
|
|
82
|
-
print('yes')
|
83
83
|
# 登录
|
84
84
|
_driver.get(_url)
|
85
85
|
time.sleep(0.1)
|
@@ -91,16 +91,16 @@ def test():
|
|
91
91
|
time.sleep(0.1)
|
92
92
|
|
93
93
|
|
94
|
-
_file = os.path.join(cookie_path,
|
94
|
+
_file = os.path.join(cookie_path, filename_aikucun)
|
95
95
|
with open(_file, 'w') as f:
|
96
96
|
# 将cookies保存为json格式
|
97
97
|
cookies_list = _driver.get_cookies()
|
98
|
-
for cookie in cookies_list:
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
98
|
+
# for cookie in cookies_list:
|
99
|
+
# # 该字段有问题所以删除就可以
|
100
|
+
# if 'expiry' in cookie:
|
101
|
+
# del cookie['expiry']
|
102
|
+
# # if 'domain' in cookie:
|
103
|
+
# # cookie['domain'] = '.taobao.com'
|
104
104
|
cookies_list = json.dumps(cookies_list)
|
105
105
|
f.write(cookies_list)
|
106
106
|
print(f'cookie已保存: {_file}')
|
@@ -109,7 +109,8 @@ def test():
|
|
109
109
|
|
110
110
|
class AikuCun:
|
111
111
|
def __init__(self):
|
112
|
-
self.url = 'https://gray-merc.aikucun.com/index.html'
|
112
|
+
# self.url = 'https://gray-merc.aikucun.com/index.html'
|
113
|
+
self.sp_url = 'https://treasurebox.aikucun.com/dashboard/commodity/ranking/merchant?LS=true&shopId=1814114991487782914&from=menu&v=0.1936043279838604'
|
113
114
|
self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
|
114
115
|
|
115
116
|
def login(self, shop_name='aikucun'):
|
@@ -173,7 +174,7 @@ class AikuCun:
|
|
173
174
|
_driver.maximize_window() # 窗口最大化 方便后续加载数据
|
174
175
|
|
175
176
|
# 登录
|
176
|
-
_driver.get(self.
|
177
|
+
_driver.get(self.sp_url)
|
177
178
|
_driver.delete_all_cookies() # 首先清除浏览器打开已有的cookies
|
178
179
|
name_lists = os.listdir(self.cookie_path) # cookie 放在主目录下的 cookies 文件夹
|
179
180
|
for name in name_lists:
|
@@ -193,9 +194,10 @@ class AikuCun:
|
|
193
194
|
"""
|
194
195
|
|
195
196
|
_driver = self.login(shop_name=shop_name)
|
196
|
-
|
197
|
-
_driver.get(
|
197
|
+
|
198
|
+
_driver.get(self.sp_url)
|
198
199
|
time.sleep(3)
|
200
|
+
# breakpoint()
|
199
201
|
|
200
202
|
today = datetime.date.today()
|
201
203
|
for date_s in range(date_num):
|
@@ -271,7 +273,7 @@ class AikuCun:
|
|
271
273
|
|
272
274
|
def akucun():
|
273
275
|
akc = AikuCun()
|
274
|
-
akc.get_data(shop_name='aikucun', date_num=
|
276
|
+
akc.get_data(shop_name='aikucun', date_num=2)
|
275
277
|
# akc.clean_data()
|
276
278
|
|
277
279
|
# 新版 数据分类
|
@@ -296,5 +298,5 @@ def akucun():
|
|
296
298
|
|
297
299
|
if __name__ == '__main__':
|
298
300
|
pass
|
299
|
-
#
|
301
|
+
# get_cookie_aikucun()
|
300
302
|
akucun()
|
@@ -1,7 +1,7 @@
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
2
2
|
mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
3
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
4
|
-
mdbq/aggregation/aggregation.py,sha256=
|
4
|
+
mdbq/aggregation/aggregation.py,sha256=CS_gMBwPKQo7uId8BrsYNTjtCZKzRVV7gT4PfE2Q46k,76014
|
5
5
|
mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
|
6
6
|
mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
|
7
7
|
mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
|
@@ -9,7 +9,7 @@ mdbq/aggregation/query_data.py,sha256=-iXeXTTdAEmFLvoOGIKB-6-moszxRCATtNe2QBzucZ
|
|
9
9
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
10
10
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
11
11
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
12
|
-
mdbq/clean/data_clean.py,sha256=
|
12
|
+
mdbq/clean/data_clean.py,sha256=QvbM_mRHvUbwvJu0UpJu4SUQNxLWsFl7QApoq6cPiVU,104788
|
13
13
|
mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
|
14
14
|
mdbq/company/copysh.py,sha256=xjlfb-RwW0oLyyHQZFUtmXvCpWI7Y_nhwa0tqW14wZk,19345
|
15
15
|
mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
|
@@ -38,10 +38,10 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
|
|
38
38
|
mdbq/pbix/refresh_all.py,sha256=viOlLCmz9zg61Q2nzjgl8dChfQxnxRd1A_jmQMb2oDM,5918
|
39
39
|
mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
|
40
40
|
mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
41
|
-
mdbq/req_post/req_tb.py,sha256=
|
41
|
+
mdbq/req_post/req_tb.py,sha256=0gVmmVewIpsJVEXkEEW0UeSSPqE9iAyLpMS0P4qjlX0,35270
|
42
42
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
43
|
-
mdbq/spider/aikucun.py,sha256=
|
44
|
-
mdbq-2.3.
|
45
|
-
mdbq-2.3.
|
46
|
-
mdbq-2.3.
|
47
|
-
mdbq-2.3.
|
43
|
+
mdbq/spider/aikucun.py,sha256=fszwS5jml5-S98u5FurBKVS2L4O-0wXFdU6CsGqbWrE,14104
|
44
|
+
mdbq-2.3.6.dist-info/METADATA,sha256=wz2ymOmXc17yN6AtYhHOLaiXhO2H5vx14ouOrvKA2dQ,245
|
45
|
+
mdbq-2.3.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
46
|
+
mdbq-2.3.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
47
|
+
mdbq-2.3.6.dist-info/RECORD,,
|
File without changes
|
File without changes
|