xtn-tools-pro 1.0.0.2.7__py3-none-any.whl → 1.0.0.2.9__py3-none-any.whl

xtn_tools_pro/task_pro/go_fun.py

@@ -8,36 +8,38 @@
 # --------------------------------------------------------------------------------------------------
 # 2025/1/14    xiatn     V00.01.000    新建
 # --------------------------------------------------------------------------------------------------
-import os
-import configparser
-from xtn_tools_pro.utils.file_utils import check_file_exists
+import time
+import inspect
+import requests
+import threading
+import multiprocessing
+import concurrent.futures
+from multiprocessing import Process
 
 
 class GoFun:
-    def __init__(self, ini_dict):
+    def __init__(self, ini_dict, logger, go_task_function=None):
         # 读取配置信息
         host = ini_dict.get('host', '')
         port = ini_dict.get('port', 0)
         task = ini_dict.get('task', '')
         auto = ini_dict.get('auto', '')
-        is_processes = ini_dict.get('is_processes', False)
         processes_num = ini_dict.get('processes_num', 0)
-        is_thread = ini_dict.get('is_thread', False)
         thread_num = ini_dict.get('thread_num', 0)
+        restart_time = ini_dict.get('restart_time', 0)
 
         self.__ini_info = {
             "host": host,
             "port": port,
             "task": task,
             "auto": auto,
-            "is_processes": is_processes,
             "processes_num": processes_num,
-            "is_thread": is_thread,
             "thread_num": thread_num,
+            "restart_time": restart_time,
         }
 
         for server_k, server_v in self.__ini_info.items():
-            if not server_v and server_k not in ["port", "is_processes", "processes_num", "is_thread", "thread_num"]:
+            if not server_v and server_k not in ["port", "processes_num", "thread_num", "restart_time"]:
                 raise Exception(f"ini_dict 配置 {server_k} 不存在")
 
         if port:
@@ -47,8 +49,181 @@ class GoFun:
 
         download_url = task_host + "/filter_server/phone/get"
         upload_url = task_host + "/filter_server/phone/update"
+        external_ip = self.__get_external_ip()
 
         self.__ini_info["download_url"] = download_url
         self.__ini_info["upload_url"] = upload_url
+        self.__ini_info["external_ip"] = external_ip
 
-        print(self.__ini_info)
+        self.logger = logger
+
+        if not go_task_function:
+            return
+
+        # 共享任务队列
+        download_queue = multiprocessing.Queue()
+        upload_queue = multiprocessing.Queue()
+        download_task_process = multiprocessing.Process(target=self._download_and_upload_task,
+                                                        args=(download_queue, upload_queue, self.__ini_info, logger))
+        download_task_process.start()
+
+        # 根据配置启动任务
+        if go_task_function:
+            go_task_fun_process = multiprocessing.Process(target=self._go_task_fun_task,
+                                                          args=(download_queue, upload_queue, self.__ini_info,
+                                                                go_task_function, logger))
+            go_task_fun_process.start()
+
+    def __get_external_ip(self):
+        """
+            获取当前网络ip
+        :return:
+        """
+        while True:
+            try:
+                rp = requests.get('https://httpbin.org/ip')
+                rp_json = rp.json()
+                print(f"当前网络ip --> {rp_json}")
+                return rp_json['origin']
+            except Exception as e:
+                pass
+
+    def _download_and_upload_task(self, download_queue, upload_queue, ini_info, logger):
+        """
+            使用两个线程 打开 获取任务、回写任务
+        :param queue:
+        :return:
+        """
+        caller = inspect.stack()[1]  # 获取调用者的调用栈信息
+        caller_name = caller.function  # 获取调用者的函数名
+        caller_class = caller.frame.f_locals.get('self', None)  # 获取调用者的类实例
+        if caller_name != "run" or caller_class is None:
+            raise Exception("错误调用")
+        thread_download_task = threading.Thread(target=self.__download_task, args=(download_queue, ini_info, logger))
+        thread_download_task.start()
+        thread_upload_task = threading.Thread(target=self.__upload_task, args=(upload_queue, ini_info, logger))
+        thread_upload_task.start()
+
+    def __download_task(self, download_queue, ini_info, logger):
+        """
+            获取任务
+        :param queue:
+        :return:
+        """
+        download_url = ini_info["download_url"]
+        external_ip = ini_info["external_ip"]
+        auto = ini_info["auto"]
+        task = ini_info["task"]
+        headers = {"Authorization": auto}
+        params = {"taskType": task}
+        while True:
+            qsize = download_queue.qsize()
+            logger.info(f"获取任务，当前队列任务数：{qsize}")
+            if download_queue.qsize() >= 10:
+                time.sleep(2)
+                continue
+            resp = requests.get(download_url, headers=headers, params=params, timeout=5)
+            json_data = resp.json()
+            result_list = json_data.get("result", [])
+            for task_item in result_list:
+                download_queue.put(task_item)
+            logger.info(f"成功获取任务个数：{len(result_list)}")
+
+    def __upload_task(self, queue, ini_info, logger):
+        """
+            回写任务
+        :return:
+        """
+        upload_url = ini_info["upload_url"]
+        external_ip = ini_info["external_ip"]
+        auto = ini_info["auto"]
+        task = ini_info["task"]
+        headers = {"Authorization": auto}
+        params = {"taskType": task}
+        while True:
+            # 判断队列是否有值
+            empty = queue.empty()
+            if empty:
+                time.sleep(2)
+                continue
+
+            # 循环全部获取队列的任务
+            result_list = []
+            try:
+                while True:
+                    result_list.append(queue.get_nowait())
+            except Exception as e:
+                pass
+
+            # 回写任务
+            data = {"result": result_list, "remoteAddr": external_ip}
+            while True:
+                try:
+                    resp = requests.post(upload_url, json=data, headers=headers, params=params, timeout=5)
+                    json_data = resp.json()
+                    logger.info(f"成功回写任务个数：{len(result_list)},{json_data}")
+                    break
+                except Exception as e:
+                    logger.critical(f"回写异常：{e}")
+
+    def _go_task_fun_task(self, download_queue, upload_queue, ini_info, go_task_function, logger):
+        """
+            单函数，根据配置启动程序
+        :param queue:
+        :return:
+        """
+        caller = inspect.stack()[1]  # 获取调用者的调用栈信息
+        caller_name = caller.function  # 获取调用者的函数名
+        caller_class = caller.frame.f_locals.get('self', None)  # 获取调用者的类实例
+        if caller_name != "run" or caller_class is None:
+            raise Exception("错误调用")
+
+        processes_num = ini_info["processes_num"]
+        thread_num = ini_info["thread_num"]
+        restart_time = ini_info["restart_time"]
+
+        processes_num = 1 if processes_num <= 0 else processes_num
+        thread_num = 1 if thread_num <= 0 else thread_num
+
+        go_task_fun_cnt = 0
+        processes_start_list = []
+        while True:
+            try:
+                if not processes_start_list:
+                    go_task_fun_cnt += 1
+                    logger.info(
+                        f"第{go_task_fun_cnt}次,进程数:{processes_num},线程数:{thread_num},等待{restart_time}秒强制下一次")
+                    for i in range(processes_num):
+                        p = Process(target=self._run_with_timeout,
+                                    args=(download_queue, upload_queue, thread_num, go_task_function))
+                        processes_start_list.append(p)
+                        p.start()
+
+                if not restart_time:
+                    # 一直执行 不退出
+                    continue
+
+                time.sleep(restart_time)
+                # 关闭所有进程
+                for p in processes_start_list:
+                    p.terminate()
+                    p.join()  # 等待进程确实结束
+                processes_start_list = []
+
+            except Exception as e:
+                pass
+
+    def _run_with_timeout(self, download_queue, upload_queue, thread_num, go_task_function):
+        caller = inspect.stack()[1]  # 获取调用者的调用栈信息
+        caller_name = caller.function  # 获取调用者的函数名
+        caller_class = caller.frame.f_locals.get('self', None)  # 获取调用者的类实例
+        if caller_name != "run" or caller_class is None:
+            raise Exception("错误调用")
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=thread_num) as executor:
+            # 提交10个函数到线程池中执行
+            futures = [executor.submit(go_task_function, download_queue, upload_queue) for _ in range(thread_num)]
+
+            # 等待所有线程完成
+            for future in concurrent.futures.as_completed(futures):
+                future.result()

{xtn_tools_pro-1.0.0.2.7.dist-info → xtn_tools_pro-1.0.0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.2.7
+Version: 1.0.0.2.9
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@gmail.com

{xtn_tools_pro-1.0.0.2.7.dist-info → xtn_tools_pro-1.0.0.2.9.dist-info}/RECORD

@@ -10,7 +10,7 @@ xtn_tools_pro/proxy/XiaoXiangProxy.py,sha256=45SsMZDc-3Ta4THX38vScjaeRBJSp420AJw
 xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmcyo,418
 xtn_tools_pro/proxy/proxy.py,sha256=No6E1pFY5yx2F4976pXPrLtq-QEVp79KupzcufjSN58,8703
 xtn_tools_pro/task_pro/__init__.py,sha256=nK3U47hWwE1H875ieEToH9r-jzXHS-PXk8cDstOvRE8,418
-xtn_tools_pro/task_pro/go_fun.py,sha256=x6RxA7wsqTfhlzlieJ3Q-sj9jz_ALanEMWTXIzWDBGE,1950
+xtn_tools_pro/task_pro/go_fun.py,sha256=sizIHXnFa_OQsYtdx5GJPH6rBAUbPCIIopdHqBHJEDI,9264
 xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
 xtn_tools_pro/utils/crypto.py,sha256=RZ5AET4udlraACWMeNF-17JiZ2R6Ahb47_j4tjkV7LE,3190
 xtn_tools_pro/utils/file_utils.py,sha256=VfdIxog4s1UW5NpKkCvQsUs9qHjLoNCnstZbnftkT4w,2046
@@ -19,8 +19,8 @@ xtn_tools_pro/utils/log.py,sha256=XZChGrKWgAz42o_gFi_d7DL28svF1OvAw7NS_xkIgnQ,10
 xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
 xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
 xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
-xtn_tools_pro-1.0.0.2.7.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.2.7.dist-info/METADATA,sha256=6oMhQBQO3wynl_-fvRDrKegTZyQqGiWe4y0UK-h-dik,455
-xtn_tools_pro-1.0.0.2.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.2.7.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.2.7.dist-info/RECORD,,
+xtn_tools_pro-1.0.0.2.9.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.2.9.dist-info/METADATA,sha256=X-2RKycjnQIprqWWv9A4TIZzRi7cYBEYB-e5SckIh4U,455
+xtn_tools_pro-1.0.0.2.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.2.9.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.2.9.dist-info/RECORD,,