xtn-tools-pro 1.0.0.4.1__tar.gz → 1.0.0.4.3__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {xtn-tools-pro-1.0.0.4.1/xtn_tools_pro.egg-info → xtn-tools-pro-1.0.0.4.3}/PKG-INFO +1 -1
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/setup.py +1 -1
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/task_pro/go_fun.py +18 -11
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/log.py +1 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3/xtn_tools_pro.egg-info}/PKG-INFO +1 -1
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/LICENSE +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/README.md +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/setup.cfg +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/__init__.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/db/MongoDB.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/db/MysqlDB.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/db/RedisDB.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/db/__init__.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/proxy/XiaoXiangProxy.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/proxy/__init__.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/proxy/proxy.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/task_pro/__init__.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/tools.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/__init__.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/crypto.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/file_utils.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/helpers.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/retry.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/sql.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro/utils/time_utils.py +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro.egg-info/SOURCES.txt +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro.egg-info/dependency_links.txt +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro.egg-info/requires.txt +0 -0
- {xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro.egg-info/top_level.txt +0 -0
@@ -105,7 +105,7 @@ class GoFun:
|
|
105
105
|
except Exception as e:
|
106
106
|
pass
|
107
107
|
|
108
|
-
def _download_and_upload_task(self, download_queue, upload_queue, proxies_dict, ini_info,
|
108
|
+
def _download_and_upload_task(self, download_queue, upload_queue, proxies_dict, ini_info,logger):
|
109
109
|
"""
|
110
110
|
使用两个线程 打开 获取任务、回写任务
|
111
111
|
:param queue:
|
@@ -116,11 +116,17 @@ class GoFun:
|
|
116
116
|
caller_class = caller.frame.f_locals.get('self', None) # 获取调用者的类实例
|
117
117
|
if caller_name != "run" or caller_class is None:
|
118
118
|
raise Exception("错误调用")
|
119
|
-
|
119
|
+
# 获取任务
|
120
|
+
thread_download_task = threading.Thread(target=self.__download_task,
|
121
|
+
args=(download_queue, ini_info, logger))
|
120
122
|
thread_download_task.start()
|
121
|
-
|
123
|
+
# 回写任务
|
124
|
+
thread_upload_task = threading.Thread(target=self.__upload_task,
|
125
|
+
args=(upload_queue, ini_info, logger))
|
122
126
|
thread_upload_task.start()
|
123
|
-
|
127
|
+
# 维护代理
|
128
|
+
thread_update_proxy = threading.Thread(target=self.__update_proxy,
|
129
|
+
args=(proxies_dict, ini_info, logger))
|
124
130
|
thread_update_proxy.start()
|
125
131
|
|
126
132
|
def __download_task(self, download_queue, ini_info, logger):
|
@@ -138,7 +144,7 @@ class GoFun:
|
|
138
144
|
while True:
|
139
145
|
try:
|
140
146
|
qsize = download_queue.qsize()
|
141
|
-
logger.info(f"
|
147
|
+
logger.info(f"当前队列剩余任务数:{qsize}")
|
142
148
|
if qsize >= 10:
|
143
149
|
time.sleep(2)
|
144
150
|
continue
|
@@ -155,9 +161,9 @@ class GoFun:
|
|
155
161
|
if not phone_item.isdigit(): # 判断是否全是整数(不包括小数点或负号)
|
156
162
|
continue
|
157
163
|
download_queue.put(task_item)
|
158
|
-
logger.warning(f"
|
164
|
+
logger.warning(f"成功获取任务个数:{len(result_list)}")
|
159
165
|
except Exception as e:
|
160
|
-
logger.critical(f"
|
166
|
+
logger.critical(f"获取任务请求异常:{e}")
|
161
167
|
time.sleep(2)
|
162
168
|
|
163
169
|
def __upload_task(self, upload_queue, ini_info, logger):
|
@@ -233,13 +239,13 @@ class GoFun:
|
|
233
239
|
status_code = resp.status_code
|
234
240
|
result_list = json_data.get("result", [])
|
235
241
|
if not result_list or status_code != 200:
|
236
|
-
logger.critical(f"
|
242
|
+
logger.critical(f"获取代理响应异常:{status_code} {len(result_list)} {json_data}")
|
237
243
|
time.sleep(2)
|
238
244
|
|
239
245
|
proxies_dict['http'] = 'http://' + random.choice(result_list)
|
240
246
|
proxies_dict['https'] = 'http://' + random.choice(result_list)
|
241
247
|
proxies_dict['status'] = True
|
242
|
-
logger.warning(f"
|
248
|
+
logger.warning(f"成功获取代理:{proxies_dict}")
|
243
249
|
|
244
250
|
if not update_proxies_time:
|
245
251
|
# 一直执行 不退出
|
@@ -248,7 +254,7 @@ class GoFun:
|
|
248
254
|
time.sleep(update_proxies_time)
|
249
255
|
proxies_dict['status'] = False
|
250
256
|
except Exception as e:
|
251
|
-
logger.critical(f"
|
257
|
+
logger.critical(f"获取代理请求异常:{e}")
|
252
258
|
time.sleep(2)
|
253
259
|
|
254
260
|
def _go_task_fun_task(self, download_queue, upload_queue, proxies_dict, ini_info, go_task_function, logger):
|
@@ -281,7 +287,8 @@ class GoFun:
|
|
281
287
|
for i in range(processes_num):
|
282
288
|
p = Process(target=self._run_with_timeout,
|
283
289
|
args=(
|
284
|
-
|
290
|
+
download_queue, upload_queue, proxies_dict, thread_num, logger,
|
291
|
+
go_task_function))
|
285
292
|
processes_start_list.append(p)
|
286
293
|
p.start()
|
287
294
|
|
@@ -154,6 +154,7 @@ class Log:
|
|
154
154
|
color_formatter = ColoredFormatter(
|
155
155
|
# '\033[1m%(log_color)s[%(now_datestr)s] | %(levelname)-8s | [%(func_name)s] - %(message)s\033[0m',
|
156
156
|
'%(log_color)s[%(now_datestr)s] | %(levelname)-8s | [%(func_name)s] - %(message)s',
|
157
|
+
# '%(log_color)s[%(now_datestr)s] | %(levelname)-8s | [%(func_name)-20s] | - %(message)s',
|
157
158
|
log_colors=log_colors) # 加粗对齐
|
158
159
|
# 设置控制台处理器的格式化器为带颜色的格式化器
|
159
160
|
console_handler.setFormatter(color_formatter)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{xtn-tools-pro-1.0.0.4.1 → xtn-tools-pro-1.0.0.4.3}/xtn_tools_pro.egg-info/dependency_links.txt
RENAMED
File without changes
|
File without changes
|
File without changes
|