qrpa 1.1.33__py3-none-any.whl → 1.1.35__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of qrpa might be problematic. Click here for more details.
- qrpa/RateLimitedSender.py +45 -45
- qrpa/__init__.py +31 -31
- qrpa/db_migrator.py +600 -600
- qrpa/feishu_bot_app.py +267 -267
- qrpa/fun_base.py +339 -339
- qrpa/fun_excel.py +3059 -3059
- qrpa/fun_file.py +318 -318
- qrpa/fun_web.py +258 -258
- qrpa/fun_win.py +198 -198
- qrpa/mysql_module/new_product_analysis_model.py +488 -0
- qrpa/mysql_module/shein_ledger_model.py +468 -468
- qrpa/mysql_module/shein_product_model.py +484 -484
- qrpa/mysql_module/shein_return_order_model.py +569 -569
- qrpa/mysql_module/shein_store_model.py +594 -0
- qrpa/shein_daily_report_model.py +375 -375
- qrpa/shein_excel.py +3125 -3125
- qrpa/shein_lib.py +3932 -3607
- qrpa/shein_mysql.py +22 -0
- qrpa/shein_sqlite.py +153 -153
- qrpa/shein_ziniao.py +529 -529
- qrpa/temu_chrome.py +56 -56
- qrpa/temu_excel.py +139 -139
- qrpa/temu_lib.py +154 -154
- qrpa/time_utils.py +882 -882
- qrpa/time_utils_example.py +243 -243
- qrpa/wxwork.py +318 -318
- {qrpa-1.1.33.dist-info → qrpa-1.1.35.dist-info}/METADATA +1 -1
- qrpa-1.1.35.dist-info/RECORD +33 -0
- qrpa-1.1.33.dist-info/RECORD +0 -31
- {qrpa-1.1.33.dist-info → qrpa-1.1.35.dist-info}/WHEEL +0 -0
- {qrpa-1.1.33.dist-info → qrpa-1.1.35.dist-info}/top_level.txt +0 -0
qrpa/shein_ziniao.py
CHANGED
|
@@ -1,529 +1,529 @@
|
|
|
1
|
-
"""
|
|
2
|
-
# 适用环境python3
|
|
3
|
-
# 紫鸟浏览器自动化操作 - 面向对象重构版本
|
|
4
|
-
"""
|
|
5
|
-
import os
|
|
6
|
-
import platform
|
|
7
|
-
import shutil
|
|
8
|
-
import time, datetime
|
|
9
|
-
import traceback
|
|
10
|
-
import uuid
|
|
11
|
-
import json
|
|
12
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
13
|
-
from typing import Literal, Optional, Callable, List, Dict, Any
|
|
14
|
-
|
|
15
|
-
import requests
|
|
16
|
-
import subprocess
|
|
17
|
-
from playwright import sync_api
|
|
18
|
-
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
|
19
|
-
|
|
20
|
-
# 使用 partial 或 lambda 固定部分参数
|
|
21
|
-
from functools import partial
|
|
22
|
-
|
|
23
|
-
from .fun_win import find_software_install_path
|
|
24
|
-
from .fun_base import log, hostname, send_exception, NetWorkIdleTimeout
|
|
25
|
-
from .fun_file import check_progress_json_ex, get_progress_json_ex, done_progress_json_ex, write_dict_to_file_ex
|
|
26
|
-
|
|
27
|
-
class ZiniaoClient:
|
|
28
|
-
"""紫鸟客户端管理类"""
|
|
29
|
-
|
|
30
|
-
def __init__(self, config):
|
|
31
|
-
self.config = config
|
|
32
|
-
self.version = "v5"
|
|
33
|
-
self.socket_port = 16851
|
|
34
|
-
self.is_windows = platform.system() == 'Windows'
|
|
35
|
-
self.is_mac = platform.system() == 'Darwin'
|
|
36
|
-
self.client_path = self._get_client_path()
|
|
37
|
-
self.user_info = self._get_user_info()
|
|
38
|
-
|
|
39
|
-
if not self.is_windows and not self.is_mac:
|
|
40
|
-
raise RuntimeError("webdriver/cdp只支持windows和mac操作系统")
|
|
41
|
-
|
|
42
|
-
def _get_client_path(self) -> str:
|
|
43
|
-
"""获取客户端路径"""
|
|
44
|
-
if self.is_windows:
|
|
45
|
-
ziniao = find_software_install_path('SuperBrowser')
|
|
46
|
-
if ziniao is None:
|
|
47
|
-
raise RuntimeError('未找到SuperBrowser安装路径')
|
|
48
|
-
return ziniao
|
|
49
|
-
else:
|
|
50
|
-
return 'ziniao'
|
|
51
|
-
|
|
52
|
-
def _get_user_info(self) -> Dict[str, str]:
|
|
53
|
-
"""获取用户登录信息"""
|
|
54
|
-
return {
|
|
55
|
-
"company" : self.config.ziniao.company,
|
|
56
|
-
"username": self.config.ziniao.username,
|
|
57
|
-
"password": self.config.ziniao.password
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
def kill_process(self):
|
|
61
|
-
"""杀紫鸟客户端进程"""
|
|
62
|
-
if self.version == "v5":
|
|
63
|
-
process_name = 'SuperBrowser.exe'
|
|
64
|
-
else:
|
|
65
|
-
process_name = 'ziniao.exe'
|
|
66
|
-
|
|
67
|
-
if self.is_windows:
|
|
68
|
-
os.system('taskkill /f /t /im ' + process_name)
|
|
69
|
-
elif self.is_mac:
|
|
70
|
-
os.system('killall ziniao')
|
|
71
|
-
time.sleep(3)
|
|
72
|
-
|
|
73
|
-
def start_browser(self):
|
|
74
|
-
"""启动客户端"""
|
|
75
|
-
try:
|
|
76
|
-
if self.is_windows:
|
|
77
|
-
cmd = [self.client_path, '--run_type=web_driver', '--show_sidb=true', '--ipc_type=http', '--port=' + str(self.socket_port)]
|
|
78
|
-
elif self.is_mac:
|
|
79
|
-
cmd = ['open', '-a', self.client_path, '--args', '--run_type=web_driver', '--ipc_type=http',
|
|
80
|
-
'--port=' + str(self.socket_port)]
|
|
81
|
-
else:
|
|
82
|
-
raise RuntimeError("不支持的操作系统")
|
|
83
|
-
|
|
84
|
-
subprocess.Popen(cmd)
|
|
85
|
-
time.sleep(5)
|
|
86
|
-
except Exception:
|
|
87
|
-
raise RuntimeError('start browser process failed')
|
|
88
|
-
|
|
89
|
-
def update_core(self):
|
|
90
|
-
"""下载所有内核,打开店铺前调用,需客户端版本5.285.7以上"""
|
|
91
|
-
data = {
|
|
92
|
-
"action" : "updateCore",
|
|
93
|
-
"requestId": str(uuid.uuid4()),
|
|
94
|
-
}
|
|
95
|
-
data.update(self.user_info)
|
|
96
|
-
|
|
97
|
-
while True:
|
|
98
|
-
result = self.send_http(data)
|
|
99
|
-
print(result)
|
|
100
|
-
if result is None:
|
|
101
|
-
print("等待客户端启动...")
|
|
102
|
-
time.sleep(2)
|
|
103
|
-
continue
|
|
104
|
-
if result.get("statusCode") is None or result.get("statusCode") == -10003:
|
|
105
|
-
print("当前版本不支持此接口,请升级客户端")
|
|
106
|
-
return
|
|
107
|
-
elif result.get("statusCode") == 0:
|
|
108
|
-
print("更新内核完成")
|
|
109
|
-
return
|
|
110
|
-
else:
|
|
111
|
-
print(f"等待更新内核: {json.dumps(result)}")
|
|
112
|
-
time.sleep(2)
|
|
113
|
-
|
|
114
|
-
def send_http(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
115
|
-
"""HTTP通讯方式"""
|
|
116
|
-
try:
|
|
117
|
-
url = f'http://127.0.0.1:{self.socket_port}'
|
|
118
|
-
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
|
|
119
|
-
return json.loads(response.text)
|
|
120
|
-
except Exception as err:
|
|
121
|
-
print(err)
|
|
122
|
-
return None
|
|
123
|
-
|
|
124
|
-
def delete_all_cache(self):
|
|
125
|
-
"""删除所有店铺缓存"""
|
|
126
|
-
if not self.is_windows:
|
|
127
|
-
return
|
|
128
|
-
local_appdata = os.getenv('LOCALAPPDATA')
|
|
129
|
-
cache_path = os.path.join(local_appdata, 'SuperBrowser')
|
|
130
|
-
if os.path.exists(cache_path):
|
|
131
|
-
shutil.rmtree(cache_path)
|
|
132
|
-
|
|
133
|
-
def delete_all_cache_with_path(self, path: str):
|
|
134
|
-
"""删除指定路径的店铺缓存"""
|
|
135
|
-
if not self.is_windows:
|
|
136
|
-
return
|
|
137
|
-
cache_path = os.path.join(path, 'SuperBrowser')
|
|
138
|
-
if os.path.exists(cache_path):
|
|
139
|
-
shutil.rmtree(cache_path)
|
|
140
|
-
|
|
141
|
-
def exit(self):
|
|
142
|
-
"""关闭客户端"""
|
|
143
|
-
data = {"action": "exit", "requestId": str(uuid.uuid4())}
|
|
144
|
-
data.update(self.user_info)
|
|
145
|
-
print('@@ get_exit...' + json.dumps(data))
|
|
146
|
-
self.send_http(data)
|
|
147
|
-
|
|
148
|
-
class ZiniaoBrowser:
|
|
149
|
-
"""紫鸟浏览器操作类"""
|
|
150
|
-
|
|
151
|
-
def __init__(self, client: ZiniaoClient, config):
|
|
152
|
-
self.client = client
|
|
153
|
-
self.config = config
|
|
154
|
-
|
|
155
|
-
def open_store(self, store_info: str, isWebDriverReadOnlyMode: int = 0,
|
|
156
|
-
isprivacy: int = 0, isHeadless: int = 0,
|
|
157
|
-
cookieTypeSave: int = 0, jsInfo: str = "") -> Dict[str, Any]:
|
|
158
|
-
"""打开店铺"""
|
|
159
|
-
request_id = str(uuid.uuid4())
|
|
160
|
-
data = {
|
|
161
|
-
"action" : "startBrowser",
|
|
162
|
-
"isWaitPluginUpdate" : 0,
|
|
163
|
-
"isHeadless" : isHeadless,
|
|
164
|
-
"requestId" : request_id,
|
|
165
|
-
"isWebDriverReadOnlyMode": isWebDriverReadOnlyMode,
|
|
166
|
-
"cookieTypeLoad" : 0,
|
|
167
|
-
"cookieTypeSave" : cookieTypeSave,
|
|
168
|
-
"runMode" : "1",
|
|
169
|
-
"isLoadUserPlugin" : False,
|
|
170
|
-
"pluginIdType" : 1,
|
|
171
|
-
"privacyMode" : isprivacy
|
|
172
|
-
}
|
|
173
|
-
data.update(self.client.user_info)
|
|
174
|
-
|
|
175
|
-
if store_info.isdigit():
|
|
176
|
-
data["browserId"] = store_info
|
|
177
|
-
else:
|
|
178
|
-
data["browserOauth"] = store_info
|
|
179
|
-
|
|
180
|
-
if len(str(jsInfo)) > 2:
|
|
181
|
-
data["injectJsInfo"] = json.dumps(jsInfo)
|
|
182
|
-
|
|
183
|
-
r = self.client.send_http(data)
|
|
184
|
-
if str(r.get("statusCode")) == "0":
|
|
185
|
-
return r
|
|
186
|
-
elif str(r.get("statusCode")) == "-10003":
|
|
187
|
-
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
|
188
|
-
raise RuntimeError("登录错误")
|
|
189
|
-
else:
|
|
190
|
-
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
|
191
|
-
raise RuntimeError("打开店铺失败")
|
|
192
|
-
|
|
193
|
-
def close_store(self, browser_oauth: str):
|
|
194
|
-
"""关闭店铺"""
|
|
195
|
-
request_id = str(uuid.uuid4())
|
|
196
|
-
data = {
|
|
197
|
-
"action" : "stopBrowser",
|
|
198
|
-
"requestId" : request_id,
|
|
199
|
-
"duplicate" : 0,
|
|
200
|
-
"browserOauth": browser_oauth
|
|
201
|
-
}
|
|
202
|
-
data.update(self.client.user_info)
|
|
203
|
-
|
|
204
|
-
r = self.client.send_http(data)
|
|
205
|
-
if str(r.get("statusCode")) == "0":
|
|
206
|
-
return r
|
|
207
|
-
elif str(r.get("statusCode")) == "-10003":
|
|
208
|
-
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
|
209
|
-
raise RuntimeError("登录错误")
|
|
210
|
-
else:
|
|
211
|
-
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
|
212
|
-
raise RuntimeError("关闭店铺失败")
|
|
213
|
-
|
|
214
|
-
def get_browser_list(self, platform_name="SHEIN-全球") -> List[Dict[str, Any]]:
|
|
215
|
-
"""获取浏览器列表"""
|
|
216
|
-
request_id = str(uuid.uuid4())
|
|
217
|
-
data = {
|
|
218
|
-
"action" : "getBrowserList",
|
|
219
|
-
"requestId": request_id
|
|
220
|
-
}
|
|
221
|
-
data.update(self.client.user_info)
|
|
222
|
-
|
|
223
|
-
r = self.client.send_http(data)
|
|
224
|
-
if str(r.get("statusCode")) == "0":
|
|
225
|
-
print(r)
|
|
226
|
-
# return r.get("browserList", [])
|
|
227
|
-
if platform_name == "1688":
|
|
228
|
-
return [site for site in r.get("browserList", []) if '1688' in site.get('tags')]
|
|
229
|
-
|
|
230
|
-
return [site for site in r.get("browserList", []) if site.get("platform_name") == platform_name]
|
|
231
|
-
elif str(r.get("statusCode")) == "-10003":
|
|
232
|
-
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
|
233
|
-
raise RuntimeError("登录错误")
|
|
234
|
-
else:
|
|
235
|
-
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
|
236
|
-
raise RuntimeError("获取浏览器列表失败")
|
|
237
|
-
|
|
238
|
-
def get_browser_context(self, playwright, port: int):
|
|
239
|
-
"""获取playwright浏览器会话"""
|
|
240
|
-
browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{port}")
|
|
241
|
-
context = browser.contexts[0]
|
|
242
|
-
return context
|
|
243
|
-
|
|
244
|
-
def open_ip_check(self, browser_context, ip_check_url: str) -> bool:
|
|
245
|
-
"""打开ip检测页检测ip是否正常"""
|
|
246
|
-
try:
|
|
247
|
-
page = browser_context.pages[0]
|
|
248
|
-
page.goto(ip_check_url)
|
|
249
|
-
success_button = page.locator('//button[contains(@class, "styles_btn--success")]')
|
|
250
|
-
success_button.wait_for(timeout=60000) # 等待查找元素60秒
|
|
251
|
-
print("ip检测成功")
|
|
252
|
-
return True
|
|
253
|
-
except PlaywrightTimeoutError:
|
|
254
|
-
print("ip检测超时")
|
|
255
|
-
return False
|
|
256
|
-
except Exception as e:
|
|
257
|
-
print("ip检测异常:" + traceback.format_exc())
|
|
258
|
-
return False
|
|
259
|
-
|
|
260
|
-
def open_launcher_page(self, browser_context, launcher_page: str, store_username: str, store_name: str, run_func: Callable, task_key: str):
|
|
261
|
-
"""打开启动页面并执行业务逻辑"""
|
|
262
|
-
page = browser_context.pages[0]
|
|
263
|
-
page.goto(launcher_page)
|
|
264
|
-
page.wait_for_timeout(3000)
|
|
265
|
-
|
|
266
|
-
run_func(page, store_username, store_name, task_key)
|
|
267
|
-
|
|
268
|
-
# 标记完成
|
|
269
|
-
done_progress_json_ex(self.config, task_key, store_name)
|
|
270
|
-
|
|
271
|
-
class ZiniaoTaskManager:
|
|
272
|
-
"""紫鸟任务管理类"""
|
|
273
|
-
|
|
274
|
-
def __init__(self, browser: ZiniaoBrowser, config):
|
|
275
|
-
self.browser = browser
|
|
276
|
-
self.config = config
|
|
277
|
-
|
|
278
|
-
def daily_cleanup_superbrowser(self, browser_id, force=False):
|
|
279
|
-
"""
|
|
280
|
-
每天删除一次SuperBrowser缓存文件夹
|
|
281
|
-
|
|
282
|
-
Args:
|
|
283
|
-
browser_id (str): 浏览器ID,如 '26986387919128'
|
|
284
|
-
"""
|
|
285
|
-
|
|
286
|
-
# 获取本地AppData路径
|
|
287
|
-
local_appdata = os.getenv('LOCALAPPDATA')
|
|
288
|
-
if not local_appdata:
|
|
289
|
-
log("错误: 无法获取LOCALAPPDATA路径")
|
|
290
|
-
return False
|
|
291
|
-
|
|
292
|
-
# 构建路径
|
|
293
|
-
cache_path = os.path.join(local_appdata, 'SuperBrowser')
|
|
294
|
-
target_folder = os.path.join(cache_path, f'User Data\\Chromium_{browser_id}')
|
|
295
|
-
flag_file = os.path.join(cache_path, f'User Data\\cleanup_flag_{browser_id}.txt')
|
|
296
|
-
|
|
297
|
-
# 检查目标文件夹是否存在
|
|
298
|
-
if not os.path.exists(target_folder):
|
|
299
|
-
log(f"目标文件夹不存在: {target_folder}")
|
|
300
|
-
return False
|
|
301
|
-
|
|
302
|
-
# 获取当前日期
|
|
303
|
-
today = datetime.date.today()
|
|
304
|
-
today_str = today.strftime('%Y-%m-%d')
|
|
305
|
-
|
|
306
|
-
# 检查标志文件
|
|
307
|
-
need_cleanup = True
|
|
308
|
-
|
|
309
|
-
if os.path.exists(flag_file):
|
|
310
|
-
try:
|
|
311
|
-
# 读取标志文件中的日期
|
|
312
|
-
with open(flag_file, 'r', encoding='utf-8') as f:
|
|
313
|
-
last_cleanup_date = f.read().strip()
|
|
314
|
-
|
|
315
|
-
# 如果是今天已经清理过,则跳过
|
|
316
|
-
if last_cleanup_date == today_str:
|
|
317
|
-
log(f"今天({today_str})已经清理过,跳过删除操作")
|
|
318
|
-
need_cleanup = False
|
|
319
|
-
|
|
320
|
-
except Exception as e:
|
|
321
|
-
log(f"读取标志文件时出错: {e}")
|
|
322
|
-
# 如果读取出错,继续执行清理
|
|
323
|
-
|
|
324
|
-
if need_cleanup or force:
|
|
325
|
-
try:
|
|
326
|
-
# 删除目标文件夹
|
|
327
|
-
log(f"正在删除文件夹: {target_folder}")
|
|
328
|
-
shutil.rmtree(target_folder)
|
|
329
|
-
log("删除成功!")
|
|
330
|
-
|
|
331
|
-
# 创建/更新标志文件
|
|
332
|
-
os.makedirs(os.path.dirname(flag_file), exist_ok=True)
|
|
333
|
-
with open(flag_file, 'w', encoding='utf-8') as f:
|
|
334
|
-
f.write(today_str)
|
|
335
|
-
|
|
336
|
-
log(f"已创建标志文件: {flag_file}")
|
|
337
|
-
return True
|
|
338
|
-
|
|
339
|
-
except Exception as e:
|
|
340
|
-
log(f"删除文件夹时出错: {e}")
|
|
341
|
-
return False
|
|
342
|
-
|
|
343
|
-
return True
|
|
344
|
-
|
|
345
|
-
def run_single_store_task(self, browser_info: Dict[str, Any],
|
|
346
|
-
run_func: Callable, task_key: str,
|
|
347
|
-
just_store_username: Optional[List[str]] = None,
|
|
348
|
-
is_skip_store: Optional[Callable] = None
|
|
349
|
-
):
|
|
350
|
-
"""运行单个店铺的任务"""
|
|
351
|
-
store_id = browser_info.get('browserOauth')
|
|
352
|
-
store_name = browser_info.get("browserName")
|
|
353
|
-
store_username = browser_info.get("store_username")
|
|
354
|
-
|
|
355
|
-
# 删除浏览器缓存,一天一删
|
|
356
|
-
browser_id = browser_info.get("browserId")
|
|
357
|
-
self.daily_cleanup_superbrowser(browser_id)
|
|
358
|
-
|
|
359
|
-
retry_count = 0
|
|
360
|
-
while True:
|
|
361
|
-
try:
|
|
362
|
-
retry_count += 1
|
|
363
|
-
# 记录店铺账号与店铺别名对应关系
|
|
364
|
-
cache_file = f'{self.config.auto_dir}/shein_store_alias.json'
|
|
365
|
-
write_dict_to_file_ex(cache_file, {store_username: store_name}, [store_username])
|
|
366
|
-
|
|
367
|
-
if is_skip_store and is_skip_store(store_username, store_name):
|
|
368
|
-
return
|
|
369
|
-
|
|
370
|
-
if just_store_username is not None:
|
|
371
|
-
if store_username not in just_store_username:
|
|
372
|
-
log(f'=================================跳过 just_store_username: {store_name},{store_username}, {just_store_username}======================================')
|
|
373
|
-
return
|
|
374
|
-
else:
|
|
375
|
-
log(f'---------------------------------命中 just_store_username: {store_name},{store_username}, {just_store_username}-------------------------------------')
|
|
376
|
-
|
|
377
|
-
if get_progress_json_ex(self.config, task_key, store_name):
|
|
378
|
-
log(f'=================================跳过 进度已完成: {task_key},{store_name},{store_username}=================================')
|
|
379
|
-
return
|
|
380
|
-
|
|
381
|
-
# 打开店铺
|
|
382
|
-
print(f"=====打开店铺:{store_name},{browser_id},{store_username}=====")
|
|
383
|
-
ret_json = self.browser.open_store(store_id)
|
|
384
|
-
print(ret_json)
|
|
385
|
-
store_id = ret_json.get("browserOauth") or ret_json.get("browserId")
|
|
386
|
-
|
|
387
|
-
# 获取playwright浏览器会话
|
|
388
|
-
with sync_api.sync_playwright() as playwright:
|
|
389
|
-
try:
|
|
390
|
-
browser_context = self.browser.get_browser_context(playwright, ret_json.get('debuggingPort'))
|
|
391
|
-
if browser_context is None:
|
|
392
|
-
print(f"=====关闭店铺:{store_name}=====")
|
|
393
|
-
self.browser.close_store(store_id)
|
|
394
|
-
return
|
|
395
|
-
|
|
396
|
-
# 获取ip检测页地址
|
|
397
|
-
ip_check_url = ret_json.get("ipDetectionPage")
|
|
398
|
-
if not ip_check_url:
|
|
399
|
-
print("ip检测页地址为空,请升级紫鸟浏览器到最新版")
|
|
400
|
-
print(f"=====关闭店铺:{store_name}=====")
|
|
401
|
-
self.browser.close_store(store_id)
|
|
402
|
-
raise RuntimeError("ip检测页地址为空")
|
|
403
|
-
|
|
404
|
-
ip_usable = self.browser.open_ip_check(browser_context, ip_check_url)
|
|
405
|
-
if ip_usable:
|
|
406
|
-
print("ip检测通过,打开店铺平台主页")
|
|
407
|
-
# 业务逻辑
|
|
408
|
-
try:
|
|
409
|
-
self.browser.open_launcher_page(browser_context, ret_json.get("launcherPage"), store_username, store_name, run_func, task_key)
|
|
410
|
-
except NetWorkIdleTimeout:
|
|
411
|
-
log('捕获到自定义错误: NetWorkIdleTimeout')
|
|
412
|
-
self.browser.close_store(store_id)
|
|
413
|
-
pass
|
|
414
|
-
|
|
415
|
-
else:
|
|
416
|
-
print("ip检测不通过,请检查")
|
|
417
|
-
except:
|
|
418
|
-
print("脚本运行异常:" + traceback.format_exc())
|
|
419
|
-
raise
|
|
420
|
-
finally:
|
|
421
|
-
print(f"=====关闭店铺:{store_name}=====")
|
|
422
|
-
self.browser.close_store(store_id)
|
|
423
|
-
break
|
|
424
|
-
except:
|
|
425
|
-
send_exception(f'第{retry_count}次运行失败,准备重新打开店铺: {store_username},{store_name},{store_id}')
|
|
426
|
-
self.daily_cleanup_superbrowser(browser_id, True)
|
|
427
|
-
if retry_count > 5:
|
|
428
|
-
break
|
|
429
|
-
|
|
430
|
-
def run_all_stores_task(self, browser_list: List[Dict[str, Any]],
|
|
431
|
-
run_func: Callable, task_key: str,
|
|
432
|
-
just_store_username: Optional[List[str]] = None,
|
|
433
|
-
is_skip_store: Optional[Callable] = None
|
|
434
|
-
):
|
|
435
|
-
"""循环运行所有店铺的任务"""
|
|
436
|
-
for browser_info in browser_list:
|
|
437
|
-
self.run_single_store_task(browser_info, run_func, task_key, just_store_username, is_skip_store)
|
|
438
|
-
|
|
439
|
-
def run_with_thread_pool(self, browser_list: List[Dict[str, Any]],
|
|
440
|
-
max_threads: int = 3, run_func: Callable = None,
|
|
441
|
-
task_key: str = None,
|
|
442
|
-
just_store_username: Optional[List[str]] = None,
|
|
443
|
-
is_skip_store: Optional[Callable] = None
|
|
444
|
-
):
|
|
445
|
-
"""使用线程池控制最大并发线程数运行任务"""
|
|
446
|
-
with ThreadPoolExecutor(max_workers=max_threads) as executor:
|
|
447
|
-
task = partial(self.run_single_store_task,
|
|
448
|
-
run_func=run_func, task_key=task_key,
|
|
449
|
-
just_store_username=just_store_username,
|
|
450
|
-
is_skip_store=is_skip_store)
|
|
451
|
-
log(f'店铺总数: {len(browser_list)}')
|
|
452
|
-
executor.map(task, browser_list)
|
|
453
|
-
|
|
454
|
-
class ZiniaoRunner:
|
|
455
|
-
"""紫鸟主运行器类"""
|
|
456
|
-
|
|
457
|
-
def __init__(self, config):
|
|
458
|
-
self.config = config
|
|
459
|
-
os.environ['auto_dir'] = self.config.auto_dir
|
|
460
|
-
os.environ['wxwork_bot_exception'] = self.config.wxwork_bot_exception
|
|
461
|
-
|
|
462
|
-
self.client = ZiniaoClient(config)
|
|
463
|
-
self.browser = ZiniaoBrowser(self.client, config)
|
|
464
|
-
self.task_manager = ZiniaoTaskManager(self.browser, config)
|
|
465
|
-
|
|
466
|
-
def execute(self, run_prepare: Optional[Callable] = None,
|
|
467
|
-
run: Optional[Callable] = None,
|
|
468
|
-
run_summary: Optional[Callable] = None,
|
|
469
|
-
run_notify: Optional[Callable] = None,
|
|
470
|
-
task_key: Optional[str] = None,
|
|
471
|
-
just_store_username: Optional[List[str]] = None,
|
|
472
|
-
is_skip_store: Optional[Callable] = None,
|
|
473
|
-
platform_name: Optional[str] = "SHEIN-全球",
|
|
474
|
-
threads_num: int = 3,
|
|
475
|
-
):
|
|
476
|
-
"""主执行入口"""
|
|
477
|
-
# 前置执行 if run_prepare:
|
|
478
|
-
run_prepare()
|
|
479
|
-
|
|
480
|
-
# 终止紫鸟客户端已启动的进程
|
|
481
|
-
self.client.kill_process()
|
|
482
|
-
|
|
483
|
-
print("=====启动客户端=====")
|
|
484
|
-
self.client.start_browser()
|
|
485
|
-
print("=====更新内核=====")
|
|
486
|
-
self.client.update_core()
|
|
487
|
-
|
|
488
|
-
# 获取店铺列表
|
|
489
|
-
print("=====获取店铺列表=====")
|
|
490
|
-
browser_list = self.browser.get_browser_list(platform_name=platform_name)
|
|
491
|
-
if not browser_list:
|
|
492
|
-
print("browser list is empty")
|
|
493
|
-
raise RuntimeError("店铺列表为空")
|
|
494
|
-
print(browser_list)
|
|
495
|
-
|
|
496
|
-
# 多线程并发执行任务
|
|
497
|
-
max_threads = 1 if (hostname().lower() == 'krrpa' or hostname().lower() == 'jyrpa') else threads_num
|
|
498
|
-
log(f'当前启用线程数: {max_threads}')
|
|
499
|
-
self.task_manager.run_with_thread_pool(browser_list, max_threads, run, task_key, just_store_username, is_skip_store)
|
|
500
|
-
|
|
501
|
-
# 任务重试逻辑
|
|
502
|
-
try_times = 0
|
|
503
|
-
while not check_progress_json_ex(self.config, task_key, just_store_username):
|
|
504
|
-
try_times += 1
|
|
505
|
-
send_exception(f'检测到任务未全部完成,再次执行: {try_times}')
|
|
506
|
-
self.task_manager.run_with_thread_pool(browser_list, max_threads, run, task_key, just_store_username, is_skip_store)
|
|
507
|
-
if try_times >= 4:
|
|
508
|
-
send_exception(f'检测到任务未全部完成,再次执行: {try_times}')
|
|
509
|
-
break
|
|
510
|
-
|
|
511
|
-
# 数据汇总
|
|
512
|
-
if not get_progress_json_ex(self.config, task_key, 'run_summary'):
|
|
513
|
-
if run_summary:
|
|
514
|
-
run_summary()
|
|
515
|
-
done_progress_json_ex(self.config, task_key, 'run_summary')
|
|
516
|
-
log('run_summary 完成')
|
|
517
|
-
|
|
518
|
-
# 发送通知
|
|
519
|
-
if not get_progress_json_ex(self.config, task_key, 'run_notify'):
|
|
520
|
-
if run_notify:
|
|
521
|
-
run_notify()
|
|
522
|
-
done_progress_json_ex(self.config, task_key, 'run_notify')
|
|
523
|
-
log('run_notify 完成')
|
|
524
|
-
|
|
525
|
-
# 关闭客户端
|
|
526
|
-
self.client.exit()
|
|
527
|
-
|
|
528
|
-
if __name__ == "__main__":
|
|
529
|
-
pass
|
|
1
|
+
"""
|
|
2
|
+
# 适用环境python3
|
|
3
|
+
# 紫鸟浏览器自动化操作 - 面向对象重构版本
|
|
4
|
+
"""
|
|
5
|
+
import os
|
|
6
|
+
import platform
|
|
7
|
+
import shutil
|
|
8
|
+
import time, datetime
|
|
9
|
+
import traceback
|
|
10
|
+
import uuid
|
|
11
|
+
import json
|
|
12
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
13
|
+
from typing import Literal, Optional, Callable, List, Dict, Any
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
import subprocess
|
|
17
|
+
from playwright import sync_api
|
|
18
|
+
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
|
19
|
+
|
|
20
|
+
# 使用 partial 或 lambda 固定部分参数
|
|
21
|
+
from functools import partial
|
|
22
|
+
|
|
23
|
+
from .fun_win import find_software_install_path
|
|
24
|
+
from .fun_base import log, hostname, send_exception, NetWorkIdleTimeout
|
|
25
|
+
from .fun_file import check_progress_json_ex, get_progress_json_ex, done_progress_json_ex, write_dict_to_file_ex
|
|
26
|
+
|
|
27
|
+
class ZiniaoClient:
|
|
28
|
+
"""紫鸟客户端管理类"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, config):
|
|
31
|
+
self.config = config
|
|
32
|
+
self.version = "v5"
|
|
33
|
+
self.socket_port = 16851
|
|
34
|
+
self.is_windows = platform.system() == 'Windows'
|
|
35
|
+
self.is_mac = platform.system() == 'Darwin'
|
|
36
|
+
self.client_path = self._get_client_path()
|
|
37
|
+
self.user_info = self._get_user_info()
|
|
38
|
+
|
|
39
|
+
if not self.is_windows and not self.is_mac:
|
|
40
|
+
raise RuntimeError("webdriver/cdp只支持windows和mac操作系统")
|
|
41
|
+
|
|
42
|
+
def _get_client_path(self) -> str:
|
|
43
|
+
"""获取客户端路径"""
|
|
44
|
+
if self.is_windows:
|
|
45
|
+
ziniao = find_software_install_path('SuperBrowser')
|
|
46
|
+
if ziniao is None:
|
|
47
|
+
raise RuntimeError('未找到SuperBrowser安装路径')
|
|
48
|
+
return ziniao
|
|
49
|
+
else:
|
|
50
|
+
return 'ziniao'
|
|
51
|
+
|
|
52
|
+
def _get_user_info(self) -> Dict[str, str]:
|
|
53
|
+
"""获取用户登录信息"""
|
|
54
|
+
return {
|
|
55
|
+
"company" : self.config.ziniao.company,
|
|
56
|
+
"username": self.config.ziniao.username,
|
|
57
|
+
"password": self.config.ziniao.password
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
def kill_process(self):
|
|
61
|
+
"""杀紫鸟客户端进程"""
|
|
62
|
+
if self.version == "v5":
|
|
63
|
+
process_name = 'SuperBrowser.exe'
|
|
64
|
+
else:
|
|
65
|
+
process_name = 'ziniao.exe'
|
|
66
|
+
|
|
67
|
+
if self.is_windows:
|
|
68
|
+
os.system('taskkill /f /t /im ' + process_name)
|
|
69
|
+
elif self.is_mac:
|
|
70
|
+
os.system('killall ziniao')
|
|
71
|
+
time.sleep(3)
|
|
72
|
+
|
|
73
|
+
def start_browser(self):
|
|
74
|
+
"""启动客户端"""
|
|
75
|
+
try:
|
|
76
|
+
if self.is_windows:
|
|
77
|
+
cmd = [self.client_path, '--run_type=web_driver', '--show_sidb=true', '--ipc_type=http', '--port=' + str(self.socket_port)]
|
|
78
|
+
elif self.is_mac:
|
|
79
|
+
cmd = ['open', '-a', self.client_path, '--args', '--run_type=web_driver', '--ipc_type=http',
|
|
80
|
+
'--port=' + str(self.socket_port)]
|
|
81
|
+
else:
|
|
82
|
+
raise RuntimeError("不支持的操作系统")
|
|
83
|
+
|
|
84
|
+
subprocess.Popen(cmd)
|
|
85
|
+
time.sleep(5)
|
|
86
|
+
except Exception:
|
|
87
|
+
raise RuntimeError('start browser process failed')
|
|
88
|
+
|
|
89
|
+
def update_core(self):
|
|
90
|
+
"""下载所有内核,打开店铺前调用,需客户端版本5.285.7以上"""
|
|
91
|
+
data = {
|
|
92
|
+
"action" : "updateCore",
|
|
93
|
+
"requestId": str(uuid.uuid4()),
|
|
94
|
+
}
|
|
95
|
+
data.update(self.user_info)
|
|
96
|
+
|
|
97
|
+
while True:
|
|
98
|
+
result = self.send_http(data)
|
|
99
|
+
print(result)
|
|
100
|
+
if result is None:
|
|
101
|
+
print("等待客户端启动...")
|
|
102
|
+
time.sleep(2)
|
|
103
|
+
continue
|
|
104
|
+
if result.get("statusCode") is None or result.get("statusCode") == -10003:
|
|
105
|
+
print("当前版本不支持此接口,请升级客户端")
|
|
106
|
+
return
|
|
107
|
+
elif result.get("statusCode") == 0:
|
|
108
|
+
print("更新内核完成")
|
|
109
|
+
return
|
|
110
|
+
else:
|
|
111
|
+
print(f"等待更新内核: {json.dumps(result)}")
|
|
112
|
+
time.sleep(2)
|
|
113
|
+
|
|
114
|
+
def send_http(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
115
|
+
"""HTTP通讯方式"""
|
|
116
|
+
try:
|
|
117
|
+
url = f'http://127.0.0.1:{self.socket_port}'
|
|
118
|
+
response = requests.post(url, json.dumps(data).encode('utf-8'), timeout=120)
|
|
119
|
+
return json.loads(response.text)
|
|
120
|
+
except Exception as err:
|
|
121
|
+
print(err)
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
def delete_all_cache(self):
|
|
125
|
+
"""删除所有店铺缓存"""
|
|
126
|
+
if not self.is_windows:
|
|
127
|
+
return
|
|
128
|
+
local_appdata = os.getenv('LOCALAPPDATA')
|
|
129
|
+
cache_path = os.path.join(local_appdata, 'SuperBrowser')
|
|
130
|
+
if os.path.exists(cache_path):
|
|
131
|
+
shutil.rmtree(cache_path)
|
|
132
|
+
|
|
133
|
+
def delete_all_cache_with_path(self, path: str):
|
|
134
|
+
"""删除指定路径的店铺缓存"""
|
|
135
|
+
if not self.is_windows:
|
|
136
|
+
return
|
|
137
|
+
cache_path = os.path.join(path, 'SuperBrowser')
|
|
138
|
+
if os.path.exists(cache_path):
|
|
139
|
+
shutil.rmtree(cache_path)
|
|
140
|
+
|
|
141
|
+
def exit(self):
|
|
142
|
+
"""关闭客户端"""
|
|
143
|
+
data = {"action": "exit", "requestId": str(uuid.uuid4())}
|
|
144
|
+
data.update(self.user_info)
|
|
145
|
+
print('@@ get_exit...' + json.dumps(data))
|
|
146
|
+
self.send_http(data)
|
|
147
|
+
|
|
148
|
+
class ZiniaoBrowser:
|
|
149
|
+
"""紫鸟浏览器操作类"""
|
|
150
|
+
|
|
151
|
+
def __init__(self, client: ZiniaoClient, config):
|
|
152
|
+
self.client = client
|
|
153
|
+
self.config = config
|
|
154
|
+
|
|
155
|
+
def open_store(self, store_info: str, isWebDriverReadOnlyMode: int = 0,
|
|
156
|
+
isprivacy: int = 0, isHeadless: int = 0,
|
|
157
|
+
cookieTypeSave: int = 0, jsInfo: str = "") -> Dict[str, Any]:
|
|
158
|
+
"""打开店铺"""
|
|
159
|
+
request_id = str(uuid.uuid4())
|
|
160
|
+
data = {
|
|
161
|
+
"action" : "startBrowser",
|
|
162
|
+
"isWaitPluginUpdate" : 0,
|
|
163
|
+
"isHeadless" : isHeadless,
|
|
164
|
+
"requestId" : request_id,
|
|
165
|
+
"isWebDriverReadOnlyMode": isWebDriverReadOnlyMode,
|
|
166
|
+
"cookieTypeLoad" : 0,
|
|
167
|
+
"cookieTypeSave" : cookieTypeSave,
|
|
168
|
+
"runMode" : "1",
|
|
169
|
+
"isLoadUserPlugin" : False,
|
|
170
|
+
"pluginIdType" : 1,
|
|
171
|
+
"privacyMode" : isprivacy
|
|
172
|
+
}
|
|
173
|
+
data.update(self.client.user_info)
|
|
174
|
+
|
|
175
|
+
if store_info.isdigit():
|
|
176
|
+
data["browserId"] = store_info
|
|
177
|
+
else:
|
|
178
|
+
data["browserOauth"] = store_info
|
|
179
|
+
|
|
180
|
+
if len(str(jsInfo)) > 2:
|
|
181
|
+
data["injectJsInfo"] = json.dumps(jsInfo)
|
|
182
|
+
|
|
183
|
+
r = self.client.send_http(data)
|
|
184
|
+
if str(r.get("statusCode")) == "0":
|
|
185
|
+
return r
|
|
186
|
+
elif str(r.get("statusCode")) == "-10003":
|
|
187
|
+
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
|
188
|
+
raise RuntimeError("登录错误")
|
|
189
|
+
else:
|
|
190
|
+
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
|
191
|
+
raise RuntimeError("打开店铺失败")
|
|
192
|
+
|
|
193
|
+
def close_store(self, browser_oauth: str):
|
|
194
|
+
"""关闭店铺"""
|
|
195
|
+
request_id = str(uuid.uuid4())
|
|
196
|
+
data = {
|
|
197
|
+
"action" : "stopBrowser",
|
|
198
|
+
"requestId" : request_id,
|
|
199
|
+
"duplicate" : 0,
|
|
200
|
+
"browserOauth": browser_oauth
|
|
201
|
+
}
|
|
202
|
+
data.update(self.client.user_info)
|
|
203
|
+
|
|
204
|
+
r = self.client.send_http(data)
|
|
205
|
+
if str(r.get("statusCode")) == "0":
|
|
206
|
+
return r
|
|
207
|
+
elif str(r.get("statusCode")) == "-10003":
|
|
208
|
+
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
|
209
|
+
raise RuntimeError("登录错误")
|
|
210
|
+
else:
|
|
211
|
+
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
|
212
|
+
raise RuntimeError("关闭店铺失败")
|
|
213
|
+
|
|
214
|
+
def get_browser_list(self, platform_name="SHEIN-全球") -> List[Dict[str, Any]]:
|
|
215
|
+
"""获取浏览器列表"""
|
|
216
|
+
request_id = str(uuid.uuid4())
|
|
217
|
+
data = {
|
|
218
|
+
"action" : "getBrowserList",
|
|
219
|
+
"requestId": request_id
|
|
220
|
+
}
|
|
221
|
+
data.update(self.client.user_info)
|
|
222
|
+
|
|
223
|
+
r = self.client.send_http(data)
|
|
224
|
+
if str(r.get("statusCode")) == "0":
|
|
225
|
+
print(r)
|
|
226
|
+
# return r.get("browserList", [])
|
|
227
|
+
if platform_name == "1688":
|
|
228
|
+
return [site for site in r.get("browserList", []) if '1688' in site.get('tags')]
|
|
229
|
+
|
|
230
|
+
return [site for site in r.get("browserList", []) if site.get("platform_name") == platform_name]
|
|
231
|
+
elif str(r.get("statusCode")) == "-10003":
|
|
232
|
+
print(f"login Err {json.dumps(r, ensure_ascii=False)}")
|
|
233
|
+
raise RuntimeError("登录错误")
|
|
234
|
+
else:
|
|
235
|
+
print(f"Fail {json.dumps(r, ensure_ascii=False)} ")
|
|
236
|
+
raise RuntimeError("获取浏览器列表失败")
|
|
237
|
+
|
|
238
|
+
def get_browser_context(self, playwright, port: int):
|
|
239
|
+
"""获取playwright浏览器会话"""
|
|
240
|
+
browser = playwright.chromium.connect_over_cdp(f"http://127.0.0.1:{port}")
|
|
241
|
+
context = browser.contexts[0]
|
|
242
|
+
return context
|
|
243
|
+
|
|
244
|
+
def open_ip_check(self, browser_context, ip_check_url: str) -> bool:
|
|
245
|
+
"""打开ip检测页检测ip是否正常"""
|
|
246
|
+
try:
|
|
247
|
+
page = browser_context.pages[0]
|
|
248
|
+
page.goto(ip_check_url)
|
|
249
|
+
success_button = page.locator('//button[contains(@class, "styles_btn--success")]')
|
|
250
|
+
success_button.wait_for(timeout=60000) # 等待查找元素60秒
|
|
251
|
+
print("ip检测成功")
|
|
252
|
+
return True
|
|
253
|
+
except PlaywrightTimeoutError:
|
|
254
|
+
print("ip检测超时")
|
|
255
|
+
return False
|
|
256
|
+
except Exception as e:
|
|
257
|
+
print("ip检测异常:" + traceback.format_exc())
|
|
258
|
+
return False
|
|
259
|
+
|
|
260
|
+
def open_launcher_page(self, browser_context, launcher_page: str, store_username: str, store_name: str, run_func: Callable, task_key: str):
|
|
261
|
+
"""打开启动页面并执行业务逻辑"""
|
|
262
|
+
page = browser_context.pages[0]
|
|
263
|
+
page.goto(launcher_page)
|
|
264
|
+
page.wait_for_timeout(3000)
|
|
265
|
+
|
|
266
|
+
run_func(page, store_username, store_name, task_key)
|
|
267
|
+
|
|
268
|
+
# 标记完成
|
|
269
|
+
done_progress_json_ex(self.config, task_key, store_name)
|
|
270
|
+
|
|
271
|
+
class ZiniaoTaskManager:
|
|
272
|
+
"""紫鸟任务管理类"""
|
|
273
|
+
|
|
274
|
+
def __init__(self, browser: ZiniaoBrowser, config):
|
|
275
|
+
self.browser = browser
|
|
276
|
+
self.config = config
|
|
277
|
+
|
|
278
|
+
def daily_cleanup_superbrowser(self, browser_id, force=False):
|
|
279
|
+
"""
|
|
280
|
+
每天删除一次SuperBrowser缓存文件夹
|
|
281
|
+
|
|
282
|
+
Args:
|
|
283
|
+
browser_id (str): 浏览器ID,如 '26986387919128'
|
|
284
|
+
"""
|
|
285
|
+
|
|
286
|
+
# 获取本地AppData路径
|
|
287
|
+
local_appdata = os.getenv('LOCALAPPDATA')
|
|
288
|
+
if not local_appdata:
|
|
289
|
+
log("错误: 无法获取LOCALAPPDATA路径")
|
|
290
|
+
return False
|
|
291
|
+
|
|
292
|
+
# 构建路径
|
|
293
|
+
cache_path = os.path.join(local_appdata, 'SuperBrowser')
|
|
294
|
+
target_folder = os.path.join(cache_path, f'User Data\\Chromium_{browser_id}')
|
|
295
|
+
flag_file = os.path.join(cache_path, f'User Data\\cleanup_flag_{browser_id}.txt')
|
|
296
|
+
|
|
297
|
+
# 检查目标文件夹是否存在
|
|
298
|
+
if not os.path.exists(target_folder):
|
|
299
|
+
log(f"目标文件夹不存在: {target_folder}")
|
|
300
|
+
return False
|
|
301
|
+
|
|
302
|
+
# 获取当前日期
|
|
303
|
+
today = datetime.date.today()
|
|
304
|
+
today_str = today.strftime('%Y-%m-%d')
|
|
305
|
+
|
|
306
|
+
# 检查标志文件
|
|
307
|
+
need_cleanup = True
|
|
308
|
+
|
|
309
|
+
if os.path.exists(flag_file):
|
|
310
|
+
try:
|
|
311
|
+
# 读取标志文件中的日期
|
|
312
|
+
with open(flag_file, 'r', encoding='utf-8') as f:
|
|
313
|
+
last_cleanup_date = f.read().strip()
|
|
314
|
+
|
|
315
|
+
# 如果是今天已经清理过,则跳过
|
|
316
|
+
if last_cleanup_date == today_str:
|
|
317
|
+
log(f"今天({today_str})已经清理过,跳过删除操作")
|
|
318
|
+
need_cleanup = False
|
|
319
|
+
|
|
320
|
+
except Exception as e:
|
|
321
|
+
log(f"读取标志文件时出错: {e}")
|
|
322
|
+
# 如果读取出错,继续执行清理
|
|
323
|
+
|
|
324
|
+
if need_cleanup or force:
|
|
325
|
+
try:
|
|
326
|
+
# 删除目标文件夹
|
|
327
|
+
log(f"正在删除文件夹: {target_folder}")
|
|
328
|
+
shutil.rmtree(target_folder)
|
|
329
|
+
log("删除成功!")
|
|
330
|
+
|
|
331
|
+
# 创建/更新标志文件
|
|
332
|
+
os.makedirs(os.path.dirname(flag_file), exist_ok=True)
|
|
333
|
+
with open(flag_file, 'w', encoding='utf-8') as f:
|
|
334
|
+
f.write(today_str)
|
|
335
|
+
|
|
336
|
+
log(f"已创建标志文件: {flag_file}")
|
|
337
|
+
return True
|
|
338
|
+
|
|
339
|
+
except Exception as e:
|
|
340
|
+
log(f"删除文件夹时出错: {e}")
|
|
341
|
+
return False
|
|
342
|
+
|
|
343
|
+
return True
|
|
344
|
+
|
|
345
|
+
def run_single_store_task(self, browser_info: Dict[str, Any],
|
|
346
|
+
run_func: Callable, task_key: str,
|
|
347
|
+
just_store_username: Optional[List[str]] = None,
|
|
348
|
+
is_skip_store: Optional[Callable] = None
|
|
349
|
+
):
|
|
350
|
+
"""运行单个店铺的任务"""
|
|
351
|
+
store_id = browser_info.get('browserOauth')
|
|
352
|
+
store_name = browser_info.get("browserName")
|
|
353
|
+
store_username = browser_info.get("store_username")
|
|
354
|
+
|
|
355
|
+
# 删除浏览器缓存,一天一删
|
|
356
|
+
browser_id = browser_info.get("browserId")
|
|
357
|
+
self.daily_cleanup_superbrowser(browser_id)
|
|
358
|
+
|
|
359
|
+
retry_count = 0
|
|
360
|
+
while True:
|
|
361
|
+
try:
|
|
362
|
+
retry_count += 1
|
|
363
|
+
# 记录店铺账号与店铺别名对应关系
|
|
364
|
+
cache_file = f'{self.config.auto_dir}/shein_store_alias.json'
|
|
365
|
+
write_dict_to_file_ex(cache_file, {store_username: store_name}, [store_username])
|
|
366
|
+
|
|
367
|
+
if is_skip_store and is_skip_store(store_username, store_name):
|
|
368
|
+
return
|
|
369
|
+
|
|
370
|
+
if just_store_username is not None:
|
|
371
|
+
if store_username not in just_store_username:
|
|
372
|
+
log(f'=================================跳过 just_store_username: {store_name},{store_username}, {just_store_username}======================================')
|
|
373
|
+
return
|
|
374
|
+
else:
|
|
375
|
+
log(f'---------------------------------命中 just_store_username: {store_name},{store_username}, {just_store_username}-------------------------------------')
|
|
376
|
+
|
|
377
|
+
if get_progress_json_ex(self.config, task_key, store_name):
|
|
378
|
+
log(f'=================================跳过 进度已完成: {task_key},{store_name},{store_username}=================================')
|
|
379
|
+
return
|
|
380
|
+
|
|
381
|
+
# 打开店铺
|
|
382
|
+
print(f"=====打开店铺:{store_name},{browser_id},{store_username}=====")
|
|
383
|
+
ret_json = self.browser.open_store(store_id)
|
|
384
|
+
print(ret_json)
|
|
385
|
+
store_id = ret_json.get("browserOauth") or ret_json.get("browserId")
|
|
386
|
+
|
|
387
|
+
# 获取playwright浏览器会话
|
|
388
|
+
with sync_api.sync_playwright() as playwright:
|
|
389
|
+
try:
|
|
390
|
+
browser_context = self.browser.get_browser_context(playwright, ret_json.get('debuggingPort'))
|
|
391
|
+
if browser_context is None:
|
|
392
|
+
print(f"=====关闭店铺:{store_name}=====")
|
|
393
|
+
self.browser.close_store(store_id)
|
|
394
|
+
return
|
|
395
|
+
|
|
396
|
+
# 获取ip检测页地址
|
|
397
|
+
ip_check_url = ret_json.get("ipDetectionPage")
|
|
398
|
+
if not ip_check_url:
|
|
399
|
+
print("ip检测页地址为空,请升级紫鸟浏览器到最新版")
|
|
400
|
+
print(f"=====关闭店铺:{store_name}=====")
|
|
401
|
+
self.browser.close_store(store_id)
|
|
402
|
+
raise RuntimeError("ip检测页地址为空")
|
|
403
|
+
|
|
404
|
+
ip_usable = self.browser.open_ip_check(browser_context, ip_check_url)
|
|
405
|
+
if ip_usable:
|
|
406
|
+
print("ip检测通过,打开店铺平台主页")
|
|
407
|
+
# 业务逻辑
|
|
408
|
+
try:
|
|
409
|
+
self.browser.open_launcher_page(browser_context, ret_json.get("launcherPage"), store_username, store_name, run_func, task_key)
|
|
410
|
+
except NetWorkIdleTimeout:
|
|
411
|
+
log('捕获到自定义错误: NetWorkIdleTimeout')
|
|
412
|
+
self.browser.close_store(store_id)
|
|
413
|
+
pass
|
|
414
|
+
|
|
415
|
+
else:
|
|
416
|
+
print("ip检测不通过,请检查")
|
|
417
|
+
except:
|
|
418
|
+
print("脚本运行异常:" + traceback.format_exc())
|
|
419
|
+
raise
|
|
420
|
+
finally:
|
|
421
|
+
print(f"=====关闭店铺:{store_name}=====")
|
|
422
|
+
self.browser.close_store(store_id)
|
|
423
|
+
break
|
|
424
|
+
except:
|
|
425
|
+
send_exception(f'第{retry_count}次运行失败,准备重新打开店铺: {store_username},{store_name},{store_id}')
|
|
426
|
+
self.daily_cleanup_superbrowser(browser_id, True)
|
|
427
|
+
if retry_count > 5:
|
|
428
|
+
break
|
|
429
|
+
|
|
430
|
+
def run_all_stores_task(self, browser_list: List[Dict[str, Any]],
|
|
431
|
+
run_func: Callable, task_key: str,
|
|
432
|
+
just_store_username: Optional[List[str]] = None,
|
|
433
|
+
is_skip_store: Optional[Callable] = None
|
|
434
|
+
):
|
|
435
|
+
"""循环运行所有店铺的任务"""
|
|
436
|
+
for browser_info in browser_list:
|
|
437
|
+
self.run_single_store_task(browser_info, run_func, task_key, just_store_username, is_skip_store)
|
|
438
|
+
|
|
439
|
+
def run_with_thread_pool(self, browser_list: List[Dict[str, Any]],
|
|
440
|
+
max_threads: int = 3, run_func: Callable = None,
|
|
441
|
+
task_key: str = None,
|
|
442
|
+
just_store_username: Optional[List[str]] = None,
|
|
443
|
+
is_skip_store: Optional[Callable] = None
|
|
444
|
+
):
|
|
445
|
+
"""使用线程池控制最大并发线程数运行任务"""
|
|
446
|
+
with ThreadPoolExecutor(max_workers=max_threads) as executor:
|
|
447
|
+
task = partial(self.run_single_store_task,
|
|
448
|
+
run_func=run_func, task_key=task_key,
|
|
449
|
+
just_store_username=just_store_username,
|
|
450
|
+
is_skip_store=is_skip_store)
|
|
451
|
+
log(f'店铺总数: {len(browser_list)}')
|
|
452
|
+
executor.map(task, browser_list)
|
|
453
|
+
|
|
454
|
+
class ZiniaoRunner:
|
|
455
|
+
"""紫鸟主运行器类"""
|
|
456
|
+
|
|
457
|
+
def __init__(self, config):
|
|
458
|
+
self.config = config
|
|
459
|
+
os.environ['auto_dir'] = self.config.auto_dir
|
|
460
|
+
os.environ['wxwork_bot_exception'] = self.config.wxwork_bot_exception
|
|
461
|
+
|
|
462
|
+
self.client = ZiniaoClient(config)
|
|
463
|
+
self.browser = ZiniaoBrowser(self.client, config)
|
|
464
|
+
self.task_manager = ZiniaoTaskManager(self.browser, config)
|
|
465
|
+
|
|
466
|
+
def execute(self, run_prepare: Optional[Callable] = None,
|
|
467
|
+
run: Optional[Callable] = None,
|
|
468
|
+
run_summary: Optional[Callable] = None,
|
|
469
|
+
run_notify: Optional[Callable] = None,
|
|
470
|
+
task_key: Optional[str] = None,
|
|
471
|
+
just_store_username: Optional[List[str]] = None,
|
|
472
|
+
is_skip_store: Optional[Callable] = None,
|
|
473
|
+
platform_name: Optional[str] = "SHEIN-全球",
|
|
474
|
+
threads_num: int = 3,
|
|
475
|
+
):
|
|
476
|
+
"""主执行入口"""
|
|
477
|
+
# 前置执行 if run_prepare:
|
|
478
|
+
run_prepare()
|
|
479
|
+
|
|
480
|
+
# 终止紫鸟客户端已启动的进程
|
|
481
|
+
self.client.kill_process()
|
|
482
|
+
|
|
483
|
+
print("=====启动客户端=====")
|
|
484
|
+
self.client.start_browser()
|
|
485
|
+
print("=====更新内核=====")
|
|
486
|
+
self.client.update_core()
|
|
487
|
+
|
|
488
|
+
# 获取店铺列表
|
|
489
|
+
print("=====获取店铺列表=====")
|
|
490
|
+
browser_list = self.browser.get_browser_list(platform_name=platform_name)
|
|
491
|
+
if not browser_list:
|
|
492
|
+
print("browser list is empty")
|
|
493
|
+
raise RuntimeError("店铺列表为空")
|
|
494
|
+
print(browser_list)
|
|
495
|
+
|
|
496
|
+
# 多线程并发执行任务
|
|
497
|
+
max_threads = 1 if (hostname().lower() == 'krrpa' or hostname().lower() == 'jyrpa') else threads_num
|
|
498
|
+
log(f'当前启用线程数: {max_threads}')
|
|
499
|
+
self.task_manager.run_with_thread_pool(browser_list, max_threads, run, task_key, just_store_username, is_skip_store)
|
|
500
|
+
|
|
501
|
+
# 任务重试逻辑
|
|
502
|
+
try_times = 0
|
|
503
|
+
while not check_progress_json_ex(self.config, task_key, just_store_username):
|
|
504
|
+
try_times += 1
|
|
505
|
+
send_exception(f'检测到任务未全部完成,再次执行: {try_times}')
|
|
506
|
+
self.task_manager.run_with_thread_pool(browser_list, max_threads, run, task_key, just_store_username, is_skip_store)
|
|
507
|
+
if try_times >= 4:
|
|
508
|
+
send_exception(f'检测到任务未全部完成,再次执行: {try_times}')
|
|
509
|
+
break
|
|
510
|
+
|
|
511
|
+
# 数据汇总
|
|
512
|
+
if not get_progress_json_ex(self.config, task_key, 'run_summary'):
|
|
513
|
+
if run_summary:
|
|
514
|
+
run_summary()
|
|
515
|
+
done_progress_json_ex(self.config, task_key, 'run_summary')
|
|
516
|
+
log('run_summary 完成')
|
|
517
|
+
|
|
518
|
+
# 发送通知
|
|
519
|
+
if not get_progress_json_ex(self.config, task_key, 'run_notify'):
|
|
520
|
+
if run_notify:
|
|
521
|
+
run_notify()
|
|
522
|
+
done_progress_json_ex(self.config, task_key, 'run_notify')
|
|
523
|
+
log('run_notify 完成')
|
|
524
|
+
|
|
525
|
+
# 关闭客户端
|
|
526
|
+
self.client.exit()
|
|
527
|
+
|
|
528
|
+
if __name__ == "__main__":
|
|
529
|
+
pass
|