xtn-tools-pro 1.0.1.3.6__py3-none-any.whl → 1.0.1.3.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xtn_tools_pro/_dlls/__init__.py +10 -0
- xtn_tools_pro/data/__init__.py +10 -0
- xtn_tools_pro/data/set_data_pro.py +10 -0
- xtn_tools_pro/task_pro/go_fun_v4.py +463 -0
- {xtn_tools_pro-1.0.1.3.6.dist-info → xtn_tools_pro-1.0.1.3.8.dist-info}/METADATA +1 -1
- {xtn_tools_pro-1.0.1.3.6.dist-info → xtn_tools_pro-1.0.1.3.8.dist-info}/RECORD +10 -6
- {xtn_tools_pro-1.0.1.3.6.dist-info → xtn_tools_pro-1.0.1.3.8.dist-info}/LICENSE +0 -0
- {xtn_tools_pro-1.0.1.3.6.dist-info → xtn_tools_pro-1.0.1.3.8.dist-info}/WHEEL +0 -0
- {xtn_tools_pro-1.0.1.3.6.dist-info → xtn_tools_pro-1.0.1.3.8.dist-info}/entry_points.txt +0 -0
- {xtn_tools_pro-1.0.1.3.6.dist-info → xtn_tools_pro-1.0.1.3.8.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
# 说明:
|
|
5
|
+
# 程序说明xxxxxxxxxxxxxxxxxxx
|
|
6
|
+
# History:
|
|
7
|
+
# Date Author Version Modification
|
|
8
|
+
# --------------------------------------------------------------------------------------------------
|
|
9
|
+
# 2025/8/20 xiatn V00.01.000 新建
|
|
10
|
+
# --------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
# 说明:
|
|
5
|
+
# 程序说明xxxxxxxxxxxxxxxxxxx
|
|
6
|
+
# History:
|
|
7
|
+
# Date Author Version Modification
|
|
8
|
+
# --------------------------------------------------------------------------------------------------
|
|
9
|
+
# 2025/10/6 xiatn V00.01.000 新建
|
|
10
|
+
# --------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
# 说明:
|
|
5
|
+
# 程序说明xxxxxxxxxxxxxxxxxxx
|
|
6
|
+
# History:
|
|
7
|
+
# Date Author Version Modification
|
|
8
|
+
# --------------------------------------------------------------------------------------------------
|
|
9
|
+
# 2025/10/6 xiatn V00.01.000 新建
|
|
10
|
+
# --------------------------------------------------------------------------------------------------
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
# 说明:
|
|
5
|
+
# 程序说明xxxxxxxxxxxxxxxxxxx
|
|
6
|
+
# History:
|
|
7
|
+
# Date Author Version Modification
|
|
8
|
+
# --------------------------------------------------------------------------------------------------
|
|
9
|
+
# 2025/1/22 xiatn V00.01.000 新建
|
|
10
|
+
# --------------------------------------------------------------------------------------------------
|
|
11
|
+
import time
|
|
12
|
+
import queue
|
|
13
|
+
import random
|
|
14
|
+
import inspect
|
|
15
|
+
import requests
|
|
16
|
+
import threading
|
|
17
|
+
import multiprocessing
|
|
18
|
+
import concurrent.futures
|
|
19
|
+
from multiprocessing import Process
|
|
20
|
+
from xtn_tools_pro.utils.time_utils import get_time_now_timestamp
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class GoFunTaskV4:
|
|
24
|
+
def __init__(self, ini_dict, logger, go_task_function):
|
|
25
|
+
self.logger = logger
|
|
26
|
+
|
|
27
|
+
# 读取配置信息
|
|
28
|
+
host = ini_dict.get('host', '') # 域名
|
|
29
|
+
port = ini_dict.get('port', 0) # 端口
|
|
30
|
+
task_type = ini_dict.get('taskType', '') # 任务
|
|
31
|
+
phone_status = ini_dict.get('phoneStatus', '') # 任务
|
|
32
|
+
x_api_key = ini_dict.get('X-Api-Key', '') # X-Api-Key
|
|
33
|
+
proxies_dict = ini_dict.get('proxies_dict', {}) # 自定义代理
|
|
34
|
+
processes_num = ini_dict.get('processes_num', 0) # 进程数
|
|
35
|
+
thread_num = ini_dict.get('thread_num', 0) # 线程数
|
|
36
|
+
restart_time = ini_dict.get('restart_time', 0) # 间隔x秒强制重启
|
|
37
|
+
restart_time = 60 * 60 if restart_time <= 0 else restart_time # 间隔x秒强制重启时间不传默认60分钟
|
|
38
|
+
update_proxies_time = ini_dict.get('update_proxies_time', 0) # 间隔x秒更新代理
|
|
39
|
+
upload_task_time = ini_dict.get('upload_task_time', 0) # 回写间隔
|
|
40
|
+
download_not_task_time = ini_dict.get('download_not_task_time', 0) # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求
|
|
41
|
+
download_task_qsize = ini_dict.get('download_task_qsize', 10) # 触发下载任务队列的最低阈值(当下载队列小于等于x时就立刻请求下载任务接口获取任务),默认10个
|
|
42
|
+
download_task_qsize = 10 if download_task_qsize < 0 else download_task_qsize # 触发下载任务队列的最低阈值(当下载队列小于等于x时就立刻请求下载任务接口获取任务),默认10个
|
|
43
|
+
|
|
44
|
+
# 默认进程数和线程数
|
|
45
|
+
processes_num = 1 if processes_num <= 0 else processes_num
|
|
46
|
+
thread_num = 1 if thread_num <= 0 else thread_num
|
|
47
|
+
|
|
48
|
+
# 拼接地址
|
|
49
|
+
if port:
|
|
50
|
+
task_host = f"http://{host}:{port}"
|
|
51
|
+
else:
|
|
52
|
+
task_host = f"http://{host}"
|
|
53
|
+
download_url = task_host + "/filter/phone/get"
|
|
54
|
+
upload_url = task_host + "/filter/phone/update"
|
|
55
|
+
update_proxy_url = task_host + f"/filter/proxy/random/get?taskType={task}&limit=1"
|
|
56
|
+
|
|
57
|
+
# 获取网络ip
|
|
58
|
+
external_ip = self.__get_external_ip()
|
|
59
|
+
|
|
60
|
+
# 全部配置信息
|
|
61
|
+
self.__ini_info = {
|
|
62
|
+
"host": host,
|
|
63
|
+
"port": int(port),
|
|
64
|
+
"taskType": task_type,
|
|
65
|
+
"phoneStatus": phone_status,
|
|
66
|
+
"X-Api-Key": x_api_key,
|
|
67
|
+
"proxies_dict": proxies_dict,
|
|
68
|
+
"processes_num": int(processes_num),
|
|
69
|
+
"thread_num": int(thread_num),
|
|
70
|
+
"restart_time": int(restart_time),
|
|
71
|
+
"update_proxies_time": int(update_proxies_time),
|
|
72
|
+
"upload_task_time": int(upload_task_time),
|
|
73
|
+
"download_url": download_url, # 获取任务地址
|
|
74
|
+
"upload_url": upload_url, # 回写任务地址
|
|
75
|
+
"update_proxy_url": update_proxy_url, # 更新代理地址
|
|
76
|
+
"external_ip": external_ip,
|
|
77
|
+
"download_not_task_time": download_not_task_time,
|
|
78
|
+
"download_task_qsize": download_task_qsize,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
logger.debug(
|
|
82
|
+
f"\n无敌框架来咯~~~当前设置配置如下:"
|
|
83
|
+
f"\n\t功能函数重启间隔:{restart_time};进程数:{processes_num};线程数:{thread_num}"
|
|
84
|
+
f"\n\t代理更新间隔:{update_proxies_time};回写间隔{upload_task_time};\n"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# 共享任务队列
|
|
88
|
+
self.download_queue = multiprocessing.Queue()
|
|
89
|
+
self.upload_queue = multiprocessing.Queue()
|
|
90
|
+
manager = multiprocessing.Manager() # 进程1
|
|
91
|
+
self.manager_info = manager.dict()
|
|
92
|
+
self.proxies_dict = proxies_dict if proxies_dict else manager.dict()
|
|
93
|
+
|
|
94
|
+
# 获取任务
|
|
95
|
+
thread_download_task = threading.Thread(target=self.__download_task,
|
|
96
|
+
args=(self.download_queue, self.manager_info, self.__ini_info, logger))
|
|
97
|
+
thread_download_task.start()
|
|
98
|
+
|
|
99
|
+
# 回写任务
|
|
100
|
+
thread_upload_task = threading.Thread(target=self.__upload_task,
|
|
101
|
+
args=(self.upload_queue, self.__ini_info, logger))
|
|
102
|
+
thread_upload_task.start()
|
|
103
|
+
|
|
104
|
+
if not self.proxies_dict:
|
|
105
|
+
pass
|
|
106
|
+
# 维护代理
|
|
107
|
+
# thread_update_proxy = threading.Thread(target=self.__update_proxy,
|
|
108
|
+
# args=(self.proxies_dict, self.manager_info, self.__ini_info, logger))
|
|
109
|
+
# thread_update_proxy.start()
|
|
110
|
+
|
|
111
|
+
# go_task_fun_cnt = 0
|
|
112
|
+
# go_task_fun_task_process = None
|
|
113
|
+
self.manager_info["gofun_kill_status"] = False # 进程kill状态,True需要kill/False无需kill
|
|
114
|
+
self.manager_info["gofun_kill_status_qz"] = False # 进程 强制 kill状态,True需要kill/False无需kill
|
|
115
|
+
self.manager_info["gofun_run_status_time"] = get_time_now_timestamp(is_time_10=True)
|
|
116
|
+
|
|
117
|
+
go_process_list = []
|
|
118
|
+
go_task_fun_cnt = 0
|
|
119
|
+
go_task_status_error = 0
|
|
120
|
+
|
|
121
|
+
while True:
|
|
122
|
+
qsize = self.download_queue.qsize()
|
|
123
|
+
if not go_process_list and self.manager_info["gofun_kill_status"] and qsize:
|
|
124
|
+
# 状态错误
|
|
125
|
+
go_task_status_error += 1
|
|
126
|
+
if go_task_status_error >= 10:
|
|
127
|
+
self.manager_info["gofun_kill_status"] = False
|
|
128
|
+
logger.critical(
|
|
129
|
+
f"状态错误 次数{go_task_status_error}进程{len(go_process_list)}状态{self.manager_info['gofun_kill_status']}队列{qsize}")
|
|
130
|
+
time.sleep(5)
|
|
131
|
+
|
|
132
|
+
if not go_process_list and not self.manager_info["gofun_kill_status"]:
|
|
133
|
+
# 未启动gofun 且 无需kill 则启动多进程多线程
|
|
134
|
+
for i in range(processes_num):
|
|
135
|
+
p = Process(target=self._run_with_timeout,
|
|
136
|
+
args=(self.download_queue, self.upload_queue, self.proxies_dict, thread_num, logger,
|
|
137
|
+
go_task_function))
|
|
138
|
+
go_process_list.append(p)
|
|
139
|
+
p.start()
|
|
140
|
+
|
|
141
|
+
self.manager_info["gofun_run_status_time"] = get_time_now_timestamp(is_time_10=True)
|
|
142
|
+
go_task_fun_cnt += 1
|
|
143
|
+
logger.info(f"第{go_task_fun_cnt}次,进程数:{processes_num},线程数:{thread_num},等待{restart_time}秒强制下一次")
|
|
144
|
+
|
|
145
|
+
elif self.manager_info["gofun_kill_status"] and go_process_list or self.manager_info[
|
|
146
|
+
"gofun_kill_status_qz"]:
|
|
147
|
+
# 需kill
|
|
148
|
+
logger.info("检测到关闭指令,正在关闭gofun进程...")
|
|
149
|
+
for p in go_process_list:
|
|
150
|
+
# p.terminate()
|
|
151
|
+
p.join()
|
|
152
|
+
go_process_list = []
|
|
153
|
+
go_task_status_error = 0
|
|
154
|
+
self.manager_info["gofun_kill_status_qz"] = False
|
|
155
|
+
logger.info("检测到关闭指令,关闭gofun成功!!!")
|
|
156
|
+
|
|
157
|
+
elif not self.manager_info["gofun_kill_status"] and go_process_list:
|
|
158
|
+
# 无需kill 且 任务进程启动 用于定时重启
|
|
159
|
+
if self.manager_info["gofun_run_status_time"] + restart_time <= get_time_now_timestamp(is_time_10=True):
|
|
160
|
+
logger.info("检测到已达强制重启间隔,正在重启...")
|
|
161
|
+
self.manager_info["gofun_kill_status"] = True
|
|
162
|
+
|
|
163
|
+
logger.info(
|
|
164
|
+
f"主线程正常...进程数:{len(go_process_list)}...gofun(True不存在|False正常):{self.manager_info['gofun_kill_status']}...qz:{self.manager_info['gofun_kill_status_qz']}")
|
|
165
|
+
time.sleep(10)
|
|
166
|
+
|
|
167
|
+
def __get_external_ip(self, retry_cnt=10):
|
|
168
|
+
"""
|
|
169
|
+
获取当前网络ip
|
|
170
|
+
:return:
|
|
171
|
+
"""
|
|
172
|
+
for _ in range(retry_cnt):
|
|
173
|
+
try:
|
|
174
|
+
rp = requests.get('https://httpbin.org/ip')
|
|
175
|
+
rp_json = rp.json()
|
|
176
|
+
self.logger.warning(f"当前网络ip --> {rp_json}")
|
|
177
|
+
return rp_json['origin']
|
|
178
|
+
except Exception as e:
|
|
179
|
+
self.logger.critical(f"[重试{_ + 1}/{retry_cnt}]获取当前网络ip报错:{e}")
|
|
180
|
+
return "0.0.0.0"
|
|
181
|
+
|
|
182
|
+
def __download_task(self, download_queue, manager_info, ini_info, logger):
|
|
183
|
+
"""
|
|
184
|
+
获取任务
|
|
185
|
+
:param queue:
|
|
186
|
+
:return:
|
|
187
|
+
"""
|
|
188
|
+
download_url = ini_info["download_url"]
|
|
189
|
+
x_api_key = ini_info["X-Api-Key"]
|
|
190
|
+
task_type = ini_info["taskType"]
|
|
191
|
+
download_not_task_time = ini_info["download_not_task_time"]
|
|
192
|
+
download_task_qsize = ini_info["download_task_qsize"]
|
|
193
|
+
headers = {"X-Api-Key": x_api_key}
|
|
194
|
+
params = {"taskType": task_type}
|
|
195
|
+
logger.warning("下载队列启动成功...")
|
|
196
|
+
download_queue_exist_cnt = 3
|
|
197
|
+
now_qsize_v = None
|
|
198
|
+
now_qsize_v_time = None
|
|
199
|
+
while True:
|
|
200
|
+
try:
|
|
201
|
+
qsize = download_queue.qsize()
|
|
202
|
+
logger.info(f"当前队列剩余任务数:{qsize}")
|
|
203
|
+
if qsize > download_task_qsize:
|
|
204
|
+
if not now_qsize_v or qsize != now_qsize_v:
|
|
205
|
+
now_qsize_v = qsize
|
|
206
|
+
now_qsize_v_time = get_time_now_timestamp(is_time_10=True)
|
|
207
|
+
|
|
208
|
+
if now_qsize_v_time + (60 * 5) < get_time_now_timestamp(is_time_10=True):
|
|
209
|
+
now_qsize_v = None
|
|
210
|
+
now_qsize_v_time = None
|
|
211
|
+
manager_info["gofun_kill_status_qz"] = True
|
|
212
|
+
|
|
213
|
+
time.sleep(download_not_task_time)
|
|
214
|
+
continue
|
|
215
|
+
resp = requests.get(download_url, headers=headers, params=params, timeout=5)
|
|
216
|
+
json_data = resp.json()
|
|
217
|
+
result_list = json_data.get("result", [])
|
|
218
|
+
if not result_list or len(result_list) <= 0:
|
|
219
|
+
# 判断任务响应是否为空
|
|
220
|
+
download_queue_exist_cnt -= 1
|
|
221
|
+
if download_queue_exist_cnt <= 0 and not manager_info["gofun_kill_status"]:
|
|
222
|
+
manager_info["gofun_kill_status"] = True
|
|
223
|
+
logger.warning("获取任务个数为0已超设置值,判断为无任务将关闭相关进程")
|
|
224
|
+
time.sleep(10)
|
|
225
|
+
continue
|
|
226
|
+
|
|
227
|
+
download_queue_exist_cnt = 10
|
|
228
|
+
manager_info["gofun_kill_status"] = False
|
|
229
|
+
|
|
230
|
+
for task_item in result_list:
|
|
231
|
+
phone_item = task_item["phone"]
|
|
232
|
+
if not phone_item.isdigit(): # 判断是否全是整数(不包括小数点或负号)
|
|
233
|
+
continue
|
|
234
|
+
download_queue.put(task_item)
|
|
235
|
+
logger.warning(f"成功获取任务个数:{len(result_list)}")
|
|
236
|
+
except Exception as e:
|
|
237
|
+
logger.critical(f"获取任务请求异常:{e}")
|
|
238
|
+
time.sleep(2)
|
|
239
|
+
|
|
240
|
+
def __upload_task(self, upload_queue, ini_info, logger):
|
|
241
|
+
"""
|
|
242
|
+
回写任务
|
|
243
|
+
:return:
|
|
244
|
+
"""
|
|
245
|
+
upload_url = ini_info["upload_url"]
|
|
246
|
+
external_ip = ini_info["external_ip"]
|
|
247
|
+
x_api_key = ini_info["X-Api-Key"]
|
|
248
|
+
task_type = ini_info["taskType"]
|
|
249
|
+
phone_status = ini_info["phoneStatus"]
|
|
250
|
+
upload_task_time = ini_info["upload_task_time"]
|
|
251
|
+
headers = {"Authorization": x_api_key}
|
|
252
|
+
logger.warning("回写队列启动成功...")
|
|
253
|
+
while True:
|
|
254
|
+
# 判断队列是否有值
|
|
255
|
+
empty = upload_queue.empty()
|
|
256
|
+
if empty:
|
|
257
|
+
time.sleep(2)
|
|
258
|
+
continue
|
|
259
|
+
|
|
260
|
+
# 循环全部获取队列的任务
|
|
261
|
+
result_list = []
|
|
262
|
+
try:
|
|
263
|
+
while True:
|
|
264
|
+
task_item = upload_queue.get_nowait()
|
|
265
|
+
taskNo = task_item["taskNo"]
|
|
266
|
+
phone = task_item["phone"]
|
|
267
|
+
isRegistered = task_item["isRegistered"]
|
|
268
|
+
countryRegion = task_item["countryRegion"]
|
|
269
|
+
full_phone = f"{countryRegion}{phone}"
|
|
270
|
+
task_item = {
|
|
271
|
+
'taskNo': taskNo,
|
|
272
|
+
'phone': full_phone,
|
|
273
|
+
'isRegistered': isRegistered,
|
|
274
|
+
'phoneStatus': phone_status,
|
|
275
|
+
}
|
|
276
|
+
result_list.append(task_item)
|
|
277
|
+
except Exception as e:
|
|
278
|
+
pass
|
|
279
|
+
# logger.critical(f"循环全部获取队列的任务{e}")
|
|
280
|
+
|
|
281
|
+
# 回写任务
|
|
282
|
+
data = {"result": result_list, "remoteAddr": external_ip, "taskType": task_type}
|
|
283
|
+
while True:
|
|
284
|
+
try:
|
|
285
|
+
resp = requests.post(upload_url, json=data, headers=headers, timeout=5)
|
|
286
|
+
json_data = resp.json()
|
|
287
|
+
# logger.warning(f"成功回写任务个数:{len(result_list)},{json_data},{data}")
|
|
288
|
+
logger.warning(f"成功回写任务个数:{len(result_list)},{json_data}")
|
|
289
|
+
break
|
|
290
|
+
except Exception as e:
|
|
291
|
+
logger.critical(f"回写异常,{len(result_list)},{e}")
|
|
292
|
+
time.sleep(2)
|
|
293
|
+
|
|
294
|
+
if not upload_task_time:
|
|
295
|
+
# 一直执行 不退出
|
|
296
|
+
continue
|
|
297
|
+
time.sleep(upload_task_time)
|
|
298
|
+
|
|
299
|
+
def __update_proxy(self, proxies_dict, manager_info, ini_info, logger):
|
|
300
|
+
"""
|
|
301
|
+
更新代理
|
|
302
|
+
:return:
|
|
303
|
+
"""
|
|
304
|
+
update_proxy_url = ini_info["update_proxy_url"]
|
|
305
|
+
auto = ini_info["auto"]
|
|
306
|
+
update_proxies_time = ini_info["update_proxies_time"]
|
|
307
|
+
headers = {"Authorization": auto}
|
|
308
|
+
|
|
309
|
+
while True:
|
|
310
|
+
try:
|
|
311
|
+
if not manager_info.get("status"):
|
|
312
|
+
resp = requests.get(update_proxy_url, headers=headers, timeout=5)
|
|
313
|
+
json_data = resp.json()
|
|
314
|
+
status_code = resp.status_code
|
|
315
|
+
result_list = json_data.get("result", [])
|
|
316
|
+
if not result_list or status_code != 200:
|
|
317
|
+
logger.critical(f"获取代理响应异常:{status_code} {json_data}")
|
|
318
|
+
time.sleep(2)
|
|
319
|
+
continue
|
|
320
|
+
|
|
321
|
+
proxies_dict['http'] = 'http://' + random.choice(result_list)
|
|
322
|
+
proxies_dict['https'] = 'http://' + random.choice(result_list)
|
|
323
|
+
manager_info['status'] = True
|
|
324
|
+
logger.warning(f"成功获取代理:{proxies_dict}")
|
|
325
|
+
|
|
326
|
+
if not update_proxies_time:
|
|
327
|
+
# 一直执行 不退出
|
|
328
|
+
time.sleep(2)
|
|
329
|
+
continue
|
|
330
|
+
|
|
331
|
+
time.sleep(update_proxies_time)
|
|
332
|
+
manager_info['status'] = False
|
|
333
|
+
except Exception as e:
|
|
334
|
+
logger.critical(f"获取代理请求异常:{e}")
|
|
335
|
+
time.sleep(2)
|
|
336
|
+
|
|
337
|
+
def _run_with_timeout(self, download_queue, upload_queue, proxies_dict, thread_num, logger, go_task_function):
|
|
338
|
+
caller = inspect.stack()[1] # 获取调用者的调用栈信息
|
|
339
|
+
caller_name = caller.function # 获取调用者的函数名
|
|
340
|
+
caller_class = caller.frame.f_locals.get('self', None) # 获取调用者的类实例
|
|
341
|
+
if caller_name != "run" or caller_class is None:
|
|
342
|
+
raise Exception("错误调用")
|
|
343
|
+
|
|
344
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=thread_num) as executor:
|
|
345
|
+
# 提交10个函数到线程池中执行
|
|
346
|
+
futures = [executor.submit(go_task_function, self, proxies_dict, logger)
|
|
347
|
+
for _ in range(thread_num)]
|
|
348
|
+
|
|
349
|
+
# 等待所有线程完成
|
|
350
|
+
for future in concurrent.futures.as_completed(futures):
|
|
351
|
+
future.result()
|
|
352
|
+
|
|
353
|
+
def get_gofun_task_status(self):
|
|
354
|
+
status = not self.manager_info["gofun_kill_status"]
|
|
355
|
+
# self.logger.debug(f"get_gofun_task_status {status}")
|
|
356
|
+
return status
|
|
357
|
+
|
|
358
|
+
def get_download_task(self, block, timeout):
|
|
359
|
+
"""
|
|
360
|
+
获取下载任务
|
|
361
|
+
:param block: 是否阻塞等待 True阻塞/False不阻塞
|
|
362
|
+
:param timeout:
|
|
363
|
+
:return:
|
|
364
|
+
error_code:1001 队列为空;
|
|
365
|
+
"""
|
|
366
|
+
try:
|
|
367
|
+
task_item = self.download_queue.get(block=block, timeout=timeout)
|
|
368
|
+
task_item["success"] = True
|
|
369
|
+
return task_item
|
|
370
|
+
except queue.Empty as e:
|
|
371
|
+
# 捕获队列为空的异常
|
|
372
|
+
# self.logger.info(f"get_download_task 获取下载任务 {download_queue, block, timeout} 报错 队列为空: {e}")
|
|
373
|
+
return {"error": "队列为空", "error_code": 1001}
|
|
374
|
+
except Exception as e:
|
|
375
|
+
self.logger.critical(f"get_download_task 获取下载任务 {self.download_queue, block, timeout} 报错 {e}")
|
|
376
|
+
return False
|
|
377
|
+
|
|
378
|
+
def get_download_task_list(self, block=False, timeout=1, get_cnt=1):
|
|
379
|
+
"""
|
|
380
|
+
获取下载任务
|
|
381
|
+
:param block: 是否阻塞等待 True阻塞/False不阻塞
|
|
382
|
+
:param timeout:
|
|
383
|
+
:param get_cnt: 获取任务的个数
|
|
384
|
+
:return:
|
|
385
|
+
error_code:1001 队列为空;
|
|
386
|
+
"""
|
|
387
|
+
task_item_list = []
|
|
388
|
+
for _ in range(get_cnt):
|
|
389
|
+
try:
|
|
390
|
+
task_item = self.download_queue.get(block=block, timeout=timeout)
|
|
391
|
+
task_item["success"] = True
|
|
392
|
+
task_item_list.append(task_item)
|
|
393
|
+
except queue.Empty as e:
|
|
394
|
+
# 捕获队列为空的异常
|
|
395
|
+
# self.logger.info(f"get_download_task 获取下载任务 {download_queue, block, timeout} 报错 队列为空: {e}")
|
|
396
|
+
# return {"error": "队列为空", "error_code": 1001}
|
|
397
|
+
break
|
|
398
|
+
except Exception as e:
|
|
399
|
+
self.logger.critical(f"get_download_task 获取下载任务 {self.download_queue, block, timeout} 报错 {e}")
|
|
400
|
+
# return False
|
|
401
|
+
break
|
|
402
|
+
|
|
403
|
+
return task_item_list
|
|
404
|
+
|
|
405
|
+
def update_upload_task(self, task_item):
|
|
406
|
+
"""
|
|
407
|
+
更新任务
|
|
408
|
+
:param task_item:
|
|
409
|
+
:return:
|
|
410
|
+
"""
|
|
411
|
+
try:
|
|
412
|
+
task_item = self.upload_queue.put(task_item)
|
|
413
|
+
return task_item
|
|
414
|
+
except Exception as e:
|
|
415
|
+
self.logger.critical(f"update_upload_task 更新任务 {self.upload_queue, task_item} 报错 {e}")
|
|
416
|
+
return False
|
|
417
|
+
|
|
418
|
+
def retry_download_task(self, task_item):
|
|
419
|
+
"""
|
|
420
|
+
更新重试任务
|
|
421
|
+
:param task_item:
|
|
422
|
+
:return:
|
|
423
|
+
"""
|
|
424
|
+
try:
|
|
425
|
+
task_item = self.download_queue.put(task_item)
|
|
426
|
+
return task_item
|
|
427
|
+
except Exception as e:
|
|
428
|
+
self.logger.critical(f"download_queue 更新重试任务 {self.upload_queue, task_item} 报错 {e}")
|
|
429
|
+
return False
|
|
430
|
+
|
|
431
|
+
def get_download_queue_size(self):
|
|
432
|
+
"""
|
|
433
|
+
获取 下载队列 大小
|
|
434
|
+
:return:
|
|
435
|
+
"""
|
|
436
|
+
qsize = self.download_queue.qsize()
|
|
437
|
+
return qsize
|
|
438
|
+
|
|
439
|
+
def get_upload_queue_size(self):
|
|
440
|
+
"""
|
|
441
|
+
获取 回写队列 大小
|
|
442
|
+
:return:
|
|
443
|
+
"""
|
|
444
|
+
qsize = self.upload_queue.qsize()
|
|
445
|
+
return qsize
|
|
446
|
+
|
|
447
|
+
def help(self):
|
|
448
|
+
help_txt = """
|
|
449
|
+
参数说明:
|
|
450
|
+
host:必填,域名
|
|
451
|
+
port:选填,端口
|
|
452
|
+
task:必填,任务类型
|
|
453
|
+
auto:必填,token
|
|
454
|
+
proxies_dict:选填,默认为空,为空时会启动获取代理接口获取代理,不为空时则不启动该接口而是一直使用用户传递过来的代理
|
|
455
|
+
update_proxies_time:选填,默认为0,间隔x秒更新代理,0则每次启动时获取一次之后就不在请求代理接口
|
|
456
|
+
processes_num:选填,进程数,默认1
|
|
457
|
+
thread_num:选填,线程数,默认1
|
|
458
|
+
restart_time:选填,间隔x秒强制重启,默认60分钟
|
|
459
|
+
upload_task_time:选填,回写间隔,默认0
|
|
460
|
+
download_not_task_time:选填,默认0,当遇到下载任务接口返回空任务时,间隔x秒再继续请求
|
|
461
|
+
download_task_qsize:选填,默认10,触发下载任务队列的最低阈值(当下载队列小于等于x时就立刻请求下载任务接口获取任务),默认10个
|
|
462
|
+
"""
|
|
463
|
+
self.logger.info(help_txt)
|
|
@@ -2,6 +2,9 @@ xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
|
|
|
2
2
|
xtn_tools_pro/tools.py,sha256=KmOOrSnjZNJ3u9vFkMO3FX0DxtlW3fnI5fL4e_raCRs,542
|
|
3
3
|
xtn_tools_pro/tools_flie.py,sha256=-FID31G8AnV2u2djRO3Ae3Ei9ld9bSBWgzwqhYzZ5JQ,1517
|
|
4
4
|
xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
|
|
5
|
+
xtn_tools_pro/_dlls/__init__.py,sha256=U3FDWOywkVaZ1sgwZdxcZZ0gpYJ5u52Ibvf8JJklbN8,418
|
|
6
|
+
xtn_tools_pro/data/__init__.py,sha256=Cg6fMz1PlSjPa1LUyQtpzvvjvFdxiky_PjxaNFr3wIo,418
|
|
7
|
+
xtn_tools_pro/data/set_data_pro.py,sha256=Cg6fMz1PlSjPa1LUyQtpzvvjvFdxiky_PjxaNFr3wIo,418
|
|
5
8
|
xtn_tools_pro/db/MongoDB.py,sha256=CljtA5NnutOPL0j2ClrCE4RXW-9N_306D3PAdRANmuk,5699
|
|
6
9
|
xtn_tools_pro/db/MysqlDB.py,sha256=SBJrcjbZdxmtTKPGwl57NthPhs4uX8J3P6o_rK01O4k,13373
|
|
7
10
|
xtn_tools_pro/db/RedisDB.py,sha256=xi5YcMP2LISASpzyxA2r2c_5GYocMKqBrdZJJtwRZMA,6493
|
|
@@ -52,6 +55,7 @@ xtn_tools_pro/task_pro/go_fun.py,sha256=hWEt2uJ9FCvJH7PhVZttS-11A7J6zbRKwX7c5YLY
|
|
|
52
55
|
xtn_tools_pro/task_pro/go_fun_air_v1.py,sha256=25gHmW5aAV6nMY9Y6-wD7uQ_wqE39dr-eaegqVbYKM8,11035
|
|
53
56
|
xtn_tools_pro/task_pro/go_fun_v2.py,sha256=SgcXgtEBGSVL1V2LyqO0z8Md2H8JZxucYrLLIwqtiLM,18489
|
|
54
57
|
xtn_tools_pro/task_pro/go_fun_v3.py,sha256=GI1OVup-izwDflAaoy_pN6xmCXat59II7v-MooOtDms,20862
|
|
58
|
+
xtn_tools_pro/task_pro/go_fun_v4.py,sha256=zvc11XkhpF4rf9WWBgUCPHAElUYRMWW2vdHGxxITYIM,21135
|
|
55
59
|
xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
|
|
56
60
|
xtn_tools_pro/utils/audio_video.py,sha256=3Im3tGOW6IbmqDcrE1SJUg6eGyLGEyq11Qu6TCutJJw,1115
|
|
57
61
|
xtn_tools_pro/utils/crypto.py,sha256=B6NC2yVxtwrbSZnngudTFRm7YRGKKEusydVVYWwxFHs,5231
|
|
@@ -66,9 +70,9 @@ xtn_tools_pro/utils/shell.py,sha256=s6sJedxLLQokcI1hF5YSD3qgGUPK0brNcP-D3-yv54I,
|
|
|
66
70
|
xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
|
|
67
71
|
xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
|
|
68
72
|
xtn_tools_pro/utils/win.py,sha256=D5_58WMKOGe9ZwJuJr2JGAWmVISwMOZUx4t7wXUCviU,3512
|
|
69
|
-
xtn_tools_pro-1.0.1.3.
|
|
70
|
-
xtn_tools_pro-1.0.1.3.
|
|
71
|
-
xtn_tools_pro-1.0.1.3.
|
|
72
|
-
xtn_tools_pro-1.0.1.3.
|
|
73
|
-
xtn_tools_pro-1.0.1.3.
|
|
74
|
-
xtn_tools_pro-1.0.1.3.
|
|
73
|
+
xtn_tools_pro-1.0.1.3.8.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
+
xtn_tools_pro-1.0.1.3.8.dist-info/METADATA,sha256=Je3m12J9Noz5Sgs4poeT63qSivSW_9Y7j01juwwyLMg,597
|
|
75
|
+
xtn_tools_pro-1.0.1.3.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
76
|
+
xtn_tools_pro-1.0.1.3.8.dist-info/entry_points.txt,sha256=t8CtXWOgw7nRDW3XNlZh8MT_P4l8EzsFnyWi5b3ovrc,68
|
|
77
|
+
xtn_tools_pro-1.0.1.3.8.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
|
|
78
|
+
xtn_tools_pro-1.0.1.3.8.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|