xtn-tools-pro 1.0.0.6.1__py3-none-any.whl → 1.0.0.6.3__py3-none-any.whl

xtn_tools_pro/task_pro/go_fun_v2.py

@@ -34,6 +34,8 @@ class GoFunTaskV2:
         restart_time = 30 * 60 if restart_time <= 0 else restart_time  # 间隔x秒强制重启时间不传默认60分钟
         update_proxies_time = ini_dict.get('update_proxies_time', 0)  # 间隔x秒更新代理
         upload_task_tine = ini_dict.get('upload_task_tine', 0)  # 回写间隔
+        download_not_task_tine = ini_dict.get('download_not_task_tine', 0)  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        download_not_task_tine = 2 if download_not_task_tine <= 0 else download_not_task_tine  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
 
         # 拼接地址
         if port:
@@ -62,6 +64,7 @@ class GoFunTaskV2:
             "upload_url": upload_url,  # 回写任务地址
             "update_proxy_url": update_proxy_url,  # 更新代理地址
             "external_ip": external_ip,
+            "download_not_task_tine": download_not_task_tine,
         }
 
         logger.debug(
@@ -116,12 +119,14 @@ class GoFunTaskV2:
         download_queue = multiprocessing.Queue()
         upload_queue = multiprocessing.Queue()
         proxies_dict = manager.dict()
+        manager_ns = manager.dict()
+        manager_ns["gofun_task_kill_status"] = False
 
         # 下载回写代理
         download_and_upload_task_process = multiprocessing.Process(target=self._download_and_upload_task,
                                                                    name="_download_and_upload_task",
                                                                    args=(ini_info, logger, download_queue,
-                                                                         upload_queue, proxies_dict)
+                                                                         upload_queue, proxies_dict, manager_ns)
                                                                    )
         download_and_upload_task_process.start()
 
@@ -135,9 +140,15 @@ class GoFunTaskV2:
 
         while True:
             logger.debug("进程正常...")
+            gofun_task_kill_status = manager_ns["gofun_task_kill_status"]
+            if gofun_task_kill_status:
+                go_task_fun_task_process.kill()
+                go_task_fun_task_process.terminate()
+                go_task_fun_task_process.join()
+
             time.sleep(10)
 
-    def _download_and_upload_task(self, ini_info, logger, download_queue, upload_queue, proxies_dict):
+    def _download_and_upload_task(self, ini_info, logger, download_queue, upload_queue, proxies_dict, manager_ns):
         """
             使用3个线程 打开 获取任务、回写任务、代理维护
         :param ini_info:
@@ -155,7 +166,7 @@ class GoFunTaskV2:
 
         # 获取任务
         thread_download_task = threading.Thread(target=self.__download_task,
-                                                args=(download_queue, ini_info, logger))
+                                                args=(download_queue, ini_info, logger, manager_ns))
         thread_download_task.start()
 
         # 回写任务
@@ -168,7 +179,7 @@ class GoFunTaskV2:
                                                args=(proxies_dict, ini_info, logger))
         thread_update_proxy.start()
 
-    def __download_task(self, download_queue, ini_info, logger):
+    def __download_task(self, download_queue, ini_info, logger, manager_ns):
         """
             获取任务
         :param queue:
@@ -179,6 +190,8 @@ class GoFunTaskV2:
         task = ini_info["task"]
         headers = {"Authorization": auto}
         params = {"taskType": task}
+
+        download_queue_exist_cnt = 10
         while True:
             try:
                 qsize = download_queue.qsize()
@@ -192,8 +205,16 @@ class GoFunTaskV2:
 
                 if len(result_list) <= 0:
                     # 判断任务响应是否为空
-                    time.sleep(2)
+                    download_queue_exist_cnt -= 1
+                    time.sleep(10)
+                    if download_queue_exist_cnt <= 0 and manager_ns["gofun_task_kill_status"] == False:
+                        manager_ns["gofun_task_kill_status"] = True
+                        logger.warning("获取任务个数为0已超设置值,判断为无任务将关闭相关进程")
                     continue
+
+                download_queue_exist_cnt = 10
+                manager_ns["gofun_task_kill_status"] = False
+
                 for task_item in result_list:
                     phone_item = task_item["phone"]
                     if not phone_item.isdigit():  # 判断是否全是整数(不包括小数点或负号)
@@ -363,14 +384,15 @@ class GoFunTaskV2:
         :param block: 是否阻塞等待 True阻塞/False不阻塞
         :param timeout:
         :return:
+        error_code:1001 队列为空;
         """
         try:
             task_item = download_queue.get(block=block, timeout=timeout)
             return task_item
         except queue.Empty as e:
             # 捕获队列为空的异常
-            self.logger.info(f"get_download_task 获取下载任务 {download_queue, block, timeout} 报错 队列为空: {e}")
-            return False
+            # self.logger.info(f"get_download_task 获取下载任务 {download_queue, block, timeout} 报错 队列为空: {e}")
+            return {"error": "队列为空", "error_code": 1001}
         except Exception as e:
             self.logger.critical(f"get_download_task 获取下载任务 {download_queue, block, timeout} 报错 {e}")
             return False

xtn_tools_pro/task_pro/go_fun_v3.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2025/1/22    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import time
+import queue
+import random
+import inspect
+import requests
+import threading
+import multiprocessing
+import concurrent.futures
+from multiprocessing import Process
+from xtn_tools_pro.utils.time_utils import get_time_now_timestamp
+
+
+class GoFunTaskV3:
+    def __init__(self, ini_dict, logger, go_task_function):
+        self.logger = logger
+
+        # 读取配置信息
+        host = ini_dict.get('host', '')  # 域名
+        port = ini_dict.get('port', 0)  # 端口
+        task = ini_dict.get('task', '')  # 任务
+        auto = ini_dict.get('auto', '')  # token
+        processes_num = ini_dict.get('processes_num', 0)  # 进程数
+        thread_num = ini_dict.get('thread_num', 0)  # 线程数
+        restart_time = ini_dict.get('restart_time', 0)  # 间隔x秒强制重启
+        restart_time = 30 * 60 if restart_time <= 0 else restart_time  # 间隔x秒强制重启时间不传默认60分钟
+        update_proxies_time = ini_dict.get('update_proxies_time', 0)  # 间隔x秒更新代理
+        upload_task_tine = ini_dict.get('upload_task_tine', 0)  # 回写间隔
+        download_not_task_tine = ini_dict.get('download_not_task_tine', 0)  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        download_not_task_tine = 2 if download_not_task_tine <= 0 else download_not_task_tine  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        processes_num = 1 if processes_num <= 0 else processes_num
+        thread_num = 1 if thread_num <= 0 else thread_num
+
+        # 拼接地址
+        if port:
+            task_host = f"http://{host}:{port}"
+        else:
+            task_host = f"http://{host}"
+        download_url = task_host + "/filter_server/phone/get"
+        upload_url = task_host + "/filter_server/phone/update"
+        update_proxy_url = task_host + f"/filter_server/proxy/random/get?taskType={task}&limit=1"
+
+        # 获取网络ip
+        external_ip = self.__get_external_ip()
+
+        # 全部配置信息
+        self.__ini_info = {
+            "host": host,
+            "port": int(port),
+            "task": task,
+            "auto": auto,
+            "processes_num": int(processes_num),
+            "thread_num": int(thread_num),
+            "restart_time": int(restart_time),
+            "update_proxies_time": int(update_proxies_time),
+            "upload_task_tine": int(upload_task_tine),
+            "download_url": download_url,  # 获取任务地址
+            "upload_url": upload_url,  # 回写任务地址
+            "update_proxy_url": update_proxy_url,  # 更新代理地址
+            "external_ip": external_ip,
+            "download_not_task_tine": download_not_task_tine,
+        }
+
+        logger.debug(
+            f"\n无敌框架来咯~~~当前设置配置如下:"
+            f"\n\t功能函数重启间隔:{restart_time};进程数:{processes_num};线程数:{thread_num}"
+            f"\n\t代理更新间隔:{update_proxies_time};回写间隔{upload_task_tine};\n"
+        )
+
+        # 共享任务队列
+        self.download_queue = multiprocessing.Queue()
+        self.upload_queue = multiprocessing.Queue()
+        manager = multiprocessing.Manager()  # 进程1
+        self.manager_info = manager.dict()
+        self.proxies_dict = manager.dict()
+
+        # 获取任务
+        thread_download_task = threading.Thread(target=self.__download_task,
+                                                args=(self.download_queue, self.manager_info, self.__ini_info, logger))
+        thread_download_task.start()
+
+        # 回写任务
+        thread_upload_task = threading.Thread(target=self.__upload_task,
+                                              args=(self.upload_queue, self.__ini_info, logger))
+        thread_upload_task.start()
+
+        # 维护代理
+        thread_update_proxy = threading.Thread(target=self.__update_proxy,
+                                               args=(self.proxies_dict, self.manager_info, self.__ini_info, logger))
+        thread_update_proxy.start()
+
+        # go_task_fun_cnt = 0
+        # go_task_fun_task_process = None
+        self.manager_info["gofun_kill_status"] = False  # 进程kill状态，True需要kill/False无需kill
+        self.manager_info["gofun_kill_th_status"] = False  # 线程kill状态，True需要kill/False无需kill
+        self.manager_info["gofun_run_status_time"] = get_time_now_timestamp(is_time_10=True)
+
+        go_process_list = []
+        go_task_fun_cnt = 0
+        while True:
+            if not go_process_list and not self.manager_info["gofun_kill_status"]:
+                # 未启动gofun 且 无需kill 则启动多进程多线程
+                for i in range(processes_num):
+                    p = Process(target=self._run_with_timeout,
+                                args=(self.download_queue, self.upload_queue, self.proxies_dict, thread_num, logger,
+                                      go_task_function))
+                    go_process_list.append(p)
+                    p.start()
+
+                self.manager_info["gofun_run_status_time"] = get_time_now_timestamp(is_time_10=True)
+                go_task_fun_cnt += 1
+                logger.info(f"第{go_task_fun_cnt}次,进程数:{processes_num},线程数:{thread_num},等待{restart_time}秒强制下一次")
+
+            elif self.manager_info["gofun_kill_status"] and go_process_list:
+                # 需kill
+                for p in go_process_list:
+                    # p.terminate()
+                    p.join()
+                go_process_list = []
+
+            elif not self.manager_info["gofun_kill_status"] and go_process_list:
+                # 无需kill 且 任务进程启动 用于定时重启
+                if self.manager_info["gofun_run_status_time"] + restart_time <= get_time_now_timestamp(is_time_10=True):
+                    self.manager_info["gofun_kill_status"] = True
+
+            logger.debug("主线程正常...")
+            time.sleep(10)
+
+    def __get_external_ip(self):
+        """
+            获取当前网络ip
+        :return:
+        """
+        while True:
+            try:
+                rp = requests.get('https://httpbin.org/ip')
+                rp_json = rp.json()
+                self.logger.warning(f"当前网络ip --> {rp_json}")
+                return rp_json['origin']
+            except Exception as e:
+                self.logger.critical(f"获取当前网络ip{e}")
+
+    def __download_task(self, download_queue, manager_info, ini_info, logger):
+        """
+            获取任务
+        :param queue:
+        :return:
+        """
+        download_url = ini_info["download_url"]
+        auto = ini_info["auto"]
+        task = ini_info["task"]
+        headers = {"Authorization": auto}
+        params = {"taskType": task}
+
+        download_queue_exist_cnt = 3
+        while True:
+            try:
+                qsize = download_queue.qsize()
+                logger.info(f"当前队列剩余任务数:{qsize}")
+                if qsize >= 10:
+                    time.sleep(2)
+                    continue
+                resp = requests.get(download_url, headers=headers, params=params, timeout=5)
+                json_data = resp.json()
+                result_list = json_data.get("result", [])
+                if len(result_list) <= 0:
+                    # 判断任务响应是否为空
+                    download_queue_exist_cnt -= 1
+                    if download_queue_exist_cnt <= 0 and not manager_info["gofun_kill_status"]:
+                        manager_info["gofun_kill_status"] = True
+                        logger.warning("获取任务个数为0已超设置值,判断为无任务将关闭相关进程")
+                    time.sleep(10)
+                    continue
+
+                download_queue_exist_cnt = 10
+                manager_info["gofun_kill_status"] = False
+
+                for task_item in result_list:
+                    phone_item = task_item["phone"]
+                    if not phone_item.isdigit():  # 判断是否全是整数(不包括小数点或负号)
+                        continue
+                    download_queue.put(task_item)
+                logger.warning(f"成功获取任务个数:{len(result_list)}")
+            except Exception as e:
+                logger.critical(f"获取任务请求异常:{e}")
+                time.sleep(2)
+
+    def __upload_task(self, upload_queue, ini_info, logger):
+        """
+            回写任务
+        :return:
+        """
+        upload_url = ini_info["upload_url"]
+        external_ip = ini_info["external_ip"]
+        auto = ini_info["auto"]
+        task = ini_info["task"]
+        upload_task_tine = ini_info["upload_task_tine"]
+        headers = {"Authorization": auto}
+        params = {"taskType": task}
+        while True:
+            # 判断队列是否有值
+            empty = upload_queue.empty()
+            if empty:
+                time.sleep(2)
+                continue
+
+            # 循环全部获取队列的任务
+            result_list = []
+            try:
+                while True:
+                    task_item = upload_queue.get_nowait()
+                    taskNo = task_item["taskNo"]
+                    phone = task_item["phone"]
+                    isRegistered = task_item["isRegistered"]
+                    country_region = task_item["country_region"]
+                    full_phone = f"{country_region}{phone}"
+                    task_item = {
+                        'taskNo': taskNo,
+                        'phone': full_phone,
+                        'isRegistered': isRegistered
+                    }
+                    result_list.append(task_item)
+            except Exception as e:
+                logger.critical(f"循环全部获取队列的任务{e}")
+
+            # 回写任务
+            data = {"result": result_list, "remoteAddr": external_ip}
+            while True:
+                try:
+                    resp = requests.post(upload_url, json=data, headers=headers, params=params, timeout=5)
+                    json_data = resp.json()
+                    # logger.warning(f"成功回写任务个数:{len(result_list)},{json_data},{data}")
+                    logger.warning(f"成功回写任务个数:{len(result_list)},{json_data}")
+                    break
+                except Exception as e:
+                    logger.critical(f"回写异常,{len(result_list)},{e}")
+                    time.sleep(2)
+
+            if not upload_task_tine:
+                # 一直执行 不退出
+                continue
+            time.sleep(upload_task_tine)
+
+    def __update_proxy(self, proxies_dict, manager_info, ini_info, logger):
+        """
+            更新代理
+        :return:
+        """
+        update_proxy_url = ini_info["update_proxy_url"]
+        auto = ini_info["auto"]
+        update_proxies_time = ini_info["update_proxies_time"]
+        headers = {"Authorization": auto}
+
+        while True:
+            try:
+                if not manager_info.get("status"):
+                    resp = requests.get(update_proxy_url, headers=headers, timeout=5)
+                    json_data = resp.json()
+                    status_code = resp.status_code
+                    result_list = json_data.get("result", [])
+                    if not result_list or status_code != 200:
+                        logger.critical(f"获取代理响应异常:{status_code} {len(result_list)} {json_data}")
+                        time.sleep(2)
+
+                    proxies_dict['http'] = 'http://' + random.choice(result_list)
+                    proxies_dict['https'] = 'http://' + random.choice(result_list)
+                    manager_info['status'] = True
+                    logger.warning(f"成功获取代理:{proxies_dict}")
+
+                if not update_proxies_time:
+                    # 一直执行 不退出
+                    time.sleep(2)
+                    continue
+
+                time.sleep(update_proxies_time)
+                manager_info['status'] = False
+            except Exception as e:
+                logger.critical(f"获取代理请求异常:{e}")
+                time.sleep(2)
+
+    def _run_with_timeout(self, download_queue, upload_queue, proxies_dict, thread_num, logger, go_task_function):
+        caller = inspect.stack()[1]  # 获取调用者的调用栈信息
+        caller_name = caller.function  # 获取调用者的函数名
+        caller_class = caller.frame.f_locals.get('self', None)  # 获取调用者的类实例
+        if caller_name != "run" or caller_class is None:
+            raise Exception("错误调用")
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=thread_num) as executor:
+            # 提交10个函数到线程池中执行
+            futures = [executor.submit(go_task_function, self, proxies_dict, logger)
+                       for _ in range(thread_num)]
+
+            # 等待所有线程完成
+            for future in concurrent.futures.as_completed(futures):
+                future.result()
+
+    def get_gofun_task_status(self):
+        status = not self.manager_info["gofun_kill_status"]
+        # self.logger.debug(f"get_gofun_task_status {status}")
+        return status
+
+    def get_download_task(self, block, timeout):
+        """
+            获取下载任务
+        :param block: 是否阻塞等待 True阻塞/False不阻塞
+        :param timeout:
+        :return:
+        error_code:1001 队列为空;
+        """
+        try:
+            task_item = self.download_queue.get(block=block, timeout=timeout)
+            task_item["success"] = True
+            return task_item
+        except queue.Empty as e:
+            # 捕获队列为空的异常
+            # self.logger.info(f"get_download_task 获取下载任务 {download_queue, block, timeout} 报错 队列为空: {e}")
+            return {"error": "队列为空", "error_code": 1001}
+        except Exception as e:
+            self.logger.critical(f"get_download_task 获取下载任务 {self.download_queue, block, timeout} 报错 {e}")
+            return False
+
+    def update_upload_task(self, task_item):
+        """
+            更新任务
+        :param task_item:
+        :return:
+        """
+        try:
+            task_item = self.upload_queue.put(task_item)
+            return task_item
+        except Exception as e:
+            self.logger.critical(f"update_upload_task 更新任务 {self.upload_queue, task_item} 报错 {e}")
+            return False

{xtn_tools_pro-1.0.0.6.1.dist-info → xtn_tools_pro-1.0.0.6.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.6.1
+Version: 1.0.0.6.3
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@gmail.com

{xtn_tools_pro-1.0.0.6.1.dist-info → xtn_tools_pro-1.0.0.6.3.dist-info}/RECORD

@@ -11,7 +11,8 @@ xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmc
 xtn_tools_pro/proxy/proxy.py,sha256=No6E1pFY5yx2F4976pXPrLtq-QEVp79KupzcufjSN58,8703
 xtn_tools_pro/task_pro/__init__.py,sha256=nK3U47hWwE1H875ieEToH9r-jzXHS-PXk8cDstOvRE8,418
 xtn_tools_pro/task_pro/go_fun.py,sha256=hWEt2uJ9FCvJH7PhVZttS-11A7J6zbRKwX7c5YLYQag,19144
-xtn_tools_pro/task_pro/go_fun_v2.py,sha256=et1rw9yXOVQtDxCi7cX-cKs925TyJWu0jbPZbZDbIgQ,17093
+xtn_tools_pro/task_pro/go_fun_v2.py,sha256=SgcXgtEBGSVL1V2LyqO0z8Md2H8JZxucYrLLIwqtiLM,18489
+xtn_tools_pro/task_pro/go_fun_v3.py,sha256=S3OW3n-BGHszHQJApK6nQwonuUaaQXo5hZGGbSHhtRU,15183
 xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
 xtn_tools_pro/utils/crypto.py,sha256=RZ5AET4udlraACWMeNF-17JiZ2R6Ahb47_j4tjkV7LE,3190
 xtn_tools_pro/utils/file_utils.py,sha256=VfdIxog4s1UW5NpKkCvQsUs9qHjLoNCnstZbnftkT4w,2046
@@ -20,8 +21,8 @@ xtn_tools_pro/utils/log.py,sha256=m0WtTWkkwtrki1ftP8vCDR8bMfK2gcfUGx5J2x2IlLQ,10
 xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
 xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
 xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
-xtn_tools_pro-1.0.0.6.1.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.6.1.dist-info/METADATA,sha256=seinLYEoBi6O46fLzxS3xfyDL8RWny_vKQlpOX7obS0,455
-xtn_tools_pro-1.0.0.6.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.6.1.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.6.1.dist-info/RECORD,,
+xtn_tools_pro-1.0.0.6.3.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.6.3.dist-info/METADATA,sha256=HRSwWVKg0eLRnOeZ7z08TXWxNVjwEZ3dHrnSinyA9Rs,455
+xtn_tools_pro-1.0.0.6.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.6.3.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.6.3.dist-info/RECORD,,