xtn-tools-pro 1.0.0.7.1__py3-none-any.whl → 1.0.0.7.3__py3-none-any.whl

xtn_tools_pro/task_pro/go_fun_v3.py

@@ -34,9 +34,11 @@ class GoFunTaskV3:
         restart_time = ini_dict.get('restart_time', 0)  # 间隔x秒强制重启
         restart_time = 30 * 60 if restart_time <= 0 else restart_time  # 间隔x秒强制重启时间不传默认60分钟
         update_proxies_time = ini_dict.get('update_proxies_time', 0)  # 间隔x秒更新代理
-        upload_task_tine = ini_dict.get('upload_task_tine', 0)  # 回写间隔
-        download_not_task_tine = ini_dict.get('download_not_task_tine', 0)  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
-        download_not_task_tine = 2 if download_not_task_tine <= 0 else download_not_task_tine  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        upload_task_time = ini_dict.get('upload_task_time', 0)  # 回写间隔
+        download_not_task_time = ini_dict.get('download_not_task_time', 0)  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        download_not_task_time = 2 if download_not_task_time <= 0 else download_not_task_time  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+
+        # 默认进程数和线程数
         processes_num = 1 if processes_num <= 0 else processes_num
         thread_num = 1 if thread_num <= 0 else thread_num
 
@@ -62,18 +64,18 @@ class GoFunTaskV3:
             "thread_num": int(thread_num),
             "restart_time": int(restart_time),
             "update_proxies_time": int(update_proxies_time),
-            "upload_task_tine": int(upload_task_tine),
+            "upload_task_time": int(upload_task_time),
             "download_url": download_url,  # 获取任务地址
             "upload_url": upload_url,  # 回写任务地址
             "update_proxy_url": update_proxy_url,  # 更新代理地址
             "external_ip": external_ip,
-            "download_not_task_tine": download_not_task_tine,
+            "download_not_task_time": download_not_task_time,
         }
 
         logger.debug(
             f"\n无敌框架来咯~~~当前设置配置如下:"
             f"\n\t功能函数重启间隔:{restart_time};进程数:{processes_num};线程数:{thread_num}"
-            f"\n\t代理更新间隔:{update_proxies_time};回写间隔{upload_task_tine};\n"
+            f"\n\t代理更新间隔:{update_proxies_time};回写间隔{upload_task_time};\n"
         )
 
         # 共享任务队列
@@ -173,6 +175,7 @@ class GoFunTaskV3:
         download_url = ini_info["download_url"]
         auto = ini_info["auto"]
         task = ini_info["task"]
+        download_not_task_time = ini_info["download_not_task_time"]
         headers = {"Authorization": auto}
         params = {"taskType": task}
 
@@ -182,7 +185,7 @@ class GoFunTaskV3:
                 qsize = download_queue.qsize()
                 logger.info(f"当前队列剩余任务数:{qsize}")
                 if qsize >= 10:
-                    time.sleep(2)
+                    time.sleep(download_not_task_time)
                     continue
                 resp = requests.get(download_url, headers=headers, params=params, timeout=5)
                 json_data = resp.json()
@@ -218,7 +221,7 @@ class GoFunTaskV3:
         external_ip = ini_info["external_ip"]
         auto = ini_info["auto"]
         task = ini_info["task"]
-        upload_task_tine = ini_info["upload_task_tine"]
+        upload_task_time = ini_info["upload_task_time"]
         headers = {"Authorization": auto}
         params = {"taskType": task}
         while True:
@@ -245,7 +248,8 @@ class GoFunTaskV3:
                     }
                     result_list.append(task_item)
             except Exception as e:
-                logger.critical(f"循环全部获取队列的任务{e}")
+                pass
+                # logger.critical(f"循环全部获取队列的任务{e}")
 
             # 回写任务
             data = {"result": result_list, "remoteAddr": external_ip}
@@ -260,10 +264,10 @@ class GoFunTaskV3:
                     logger.critical(f"回写异常,{len(result_list)},{e}")
                     time.sleep(2)
 
-            if not upload_task_tine:
+            if not upload_task_time:
                 # 一直执行 不退出
                 continue
-            time.sleep(upload_task_tine)
+            time.sleep(upload_task_time)
 
     def __update_proxy(self, proxies_dict, manager_info, ini_info, logger):
         """

xtn_tools_pro/utils/crypto.py

@@ -8,8 +8,11 @@
 # --------------------------------------------------------------------------------------------------
 # 2024/5/13    xiatn     V00.01.000    新建
 # --------------------------------------------------------------------------------------------------
+import jwt
 import base64
 import hashlib
+import secrets
+import datetime
 
 
 def get_md5_32(s: str, is_upper=False):
@@ -117,5 +120,33 @@ def get_base64_encode(s: str):
     return encoded_string
 
 
+def generate_bearer_token(secret_key, expiration=0, algorithm='HS256', **kwargs):
+    """
+    生成 Bearer Token
+    :param secret_key: 用于加密的密钥
+    :param expiration: 过期时间戳,0则表示不过期
+    :param algorithm: 使用的加密算法（默认是 HS256）HS256、HS384、HS512
+    :param kwargs: 其他自定义的 Token 数据
+    :return: 生成的 Bearer Token
+    """
+    payload = {
+        **kwargs  # 将其他参数添加到 payload
+    }
+    if expiration:
+        payload['exp'] = expiration
+
+    token = jwt.encode(payload, secret_key, algorithm=algorithm)
+    return token
+
+
+def generate_secret_key(length=32):
+    """
+    生成一个随机的 secret key
+    :param length: key 的长度，默认为 32
+    :return: 生成的 secret key
+    """
+    return secrets.token_hex(length)
+
+
 if __name__ == '__main__':
     print(get_base64_encode(''))

xtn_tools_pro/utils/file_utils.py

@@ -58,6 +58,28 @@ def check_file_exists(file_path):
     return False
 
 
+def is_dir(dir_path):
+    """
+        传入文件夹判断是否为文件夹或文件夹是否存在
+        传入的如果是文件路径则返回False
+        传入的如果是一个不存在的文件夹则返回False
+    :param dir_path:
+    :return:
+    """
+    if not os.path.isdir(dir_path):
+        return False
+    return True
+
+
+def get_listdir(dir_path):
+    """
+        获取指定文件夹下的所有文件
+    :param dir_path:
+    :return:
+    """
+    return os.listdir(dir_path)
+
+
 if __name__ == '__main__':
     pass
     print(get_file_extension('file/2024-04-19/BOSCH GEX 125-1A/125-1AE砂磨机操作说明书:[1]_jingyan.txt'))

xtn_tools_pro/utils/log.py

@@ -107,7 +107,8 @@ class Log:
             # 记录日志文件的路径模板，用于之后创建新的日志文件
             self.path_template = self.path_template = f"{save_time_log_path}/{{date}}/{name}.log"
             path = self.path_template.format(date=self.current_date)
-            mkdirs_dir(self.path_template.format(date=self.current_date))
+            if is_write_to_file:
+                mkdirs_dir(self.path_template.format(date=self.current_date))
 
         # 创建日志格式化器
         # formatter = logging.Formatter('[%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s') # 原

xtn_tools_pro/utils/set_data.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    大文件去重
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2025/3/11    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import os
+import fnmatch
+import hashlib
+from tqdm import tqdm
+from xtn_tools_pro.utils.log import Log
+from xtn_tools_pro.utils.helpers import get_orderId_random
+from xtn_tools_pro.utils.file_utils import mkdirs_dir, get_file_extension,is_dir,get_listdir
+
+
+class PppSetDataObj:
+    def __init__(self):
+        # 随机生成一个临时文件夹
+        self.__order_id = get_orderId_random()
+        temp_dir_name = f"temp_{self.__order_id}\\"
+        now_current_working_dir = os.getcwd()
+        self.__now_current_working_dir = os.path.join(now_current_working_dir, temp_dir_name)
+        mkdirs_dir(self.__now_current_working_dir)
+
+        self.__logger = Log('set_data', './xxx.log', log_level='DEBUG', is_write_to_console=True,
+                            is_write_to_file=False,
+                            color=True, mode='a', save_time_log_path='./logs')
+
+    def set_file_data_air(self, set_file_path, num_shards=1000):
+        """
+            对单个文件去重，air版本，不对文件做任何修改，去重任何数据
+        :param set_file_path:单文件路径
+        :param num_shards:临时文件切片，推荐：数据越大值越大 10、100、1000、10000
+        :return:
+        """
+        if get_file_extension(set_file_path) != ".txt":
+            self.__logger.critical("文件不合法，只接受.txt文件")
+            return
+        self.__logger.info("正在读取文件总行数...")
+
+        with open(set_file_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"读取文件完成,总行数为:{line_count}")
+
+        num_shards = 3000 if num_shards >= 3000 else num_shards
+        num_shards = 3000 if line_count >= 30000000 else num_shards
+        num_shards = 1000 if num_shards <= 0 else num_shards
+
+        shard_file_obj_list = []
+        shard_path_list = []
+        for _ in range(num_shards):
+            shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{_}.tmp')}"
+            shard_path_list.append(shard_path)
+            shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))
+
+        with open(set_file_path, "r", encoding="utf-8") as f_r:
+            tqdm_f = tqdm(f_r, total=line_count, desc="正在去重(1/2)", unit="lines")
+            for idx, line_i in enumerate(tqdm_f):
+                line = line_i.strip().encode()
+                line_hash = hashlib.md5(line).hexdigest()
+                shard_id = int(line_hash, 16) % num_shards
+                shard_file_obj_list[shard_id].write(line_i)
+
+        for shard_file_obj in shard_file_obj_list:
+            shard_file_obj.close()
+
+        result_w_path = os.path.join(self.__now_current_working_dir, "000_去重结果.txt")
+        tqdm_f = tqdm(shard_path_list, total=len(shard_path_list), desc="正在去重(2/2)", unit="lines")
+        with open(result_w_path, "w", encoding="utf-8") as f_w:
+            for shard_path in tqdm_f:
+                with open(shard_path, "r", encoding="utf-8") as f_r:
+                    seen_list = []
+                    for line_i in f_r.readlines():
+                        line = line_i.strip()
+                        seen_list.append(line)
+                    seen_list = list(set(seen_list))
+                    w_txt = "\n".join(seen_list)
+                    f_w.write(w_txt + "\n")
+                os.remove(shard_path)  # 删除临时文件
+
+        with open(result_w_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")
+
+    def set_file_data_pro(self, set_file_dir_path, num_shards=1000):
+        """
+            对文件夹下的所有txt文件去重，pro版本，不对文件做任何修改，去重任何数据
+        :param set_file_dir_path:文件夹路径
+        :param num_shards:临时文件切片，推荐：数据越大值越大 10、100、1000、10000
+        :return:
+        """
+        if not is_dir(set_file_dir_path):
+            self.__logger.critical("文件夹不存在或不合法")
+            return
+
+        self.__logger.info("正在统计文件可去重数量...")
+        set_file_path_list = []
+        for set_file_name in get_listdir(set_file_dir_path):
+            if fnmatch.fnmatch(set_file_name, '*.txt'):
+                set_file_path_list.append(os.path.join(set_file_dir_path,set_file_name))
+        self.__logger.info(f"当前文件夹下可去重文件数量为:{len(set_file_path_list)}")
+
+        for set_file_path in set_file_path_list:
+            pass
+            # with open(set_file_path, "r", encoding="utf-8") as fp_r:
+            #     line_count = sum(1 for _ in fp_r)
+            # self.__logger.info(f"读取文件完成,总行数为：{line_count}")
+
+
+
+
+
+        # num_shards = 3000 if num_shards >= 3000 else num_shards
+        # num_shards = 3000 if line_count >= 30000000 else num_shards
+        # num_shards = 1000 if num_shards <= 0 else num_shards
+        #
+        # shard_file_obj_list = []
+        # shard_path_list = []
+        # for _ in range(num_shards):
+        #     shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{_}.tmp')}"
+        #     shard_path_list.append(shard_path)
+        #     shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))
+        #
+        # with open(set_file_path, "r", encoding="utf-8") as f_r:
+        #     tqdm_f = tqdm(f_r, total=line_count, desc="正在去重(1/2)", unit="lines")
+        #     for idx, line_i in enumerate(tqdm_f):
+        #         line = line_i.strip().encode()
+        #         line_hash = hashlib.md5(line).hexdigest()
+        #         shard_id = int(line_hash, 16) % num_shards
+        #         shard_file_obj_list[shard_id].write(line_i)
+        #
+        # for shard_file_obj in shard_file_obj_list:
+        #     shard_file_obj.close()
+        #
+        # result_w_path = os.path.join(self.__now_current_working_dir, "000_去重结果.txt")
+        # tqdm_f = tqdm(shard_path_list, total=len(shard_path_list), desc="正在去重(2/2)", unit="lines")
+        # with open(result_w_path, "w", encoding="utf-8") as f_w:
+        #     for shard_path in tqdm_f:
+        #         with open(shard_path, "r", encoding="utf-8") as f_r:
+        #             seen_list = []
+        #             for line_i in f_r.readlines():
+        #                 line = line_i.strip()
+        #                 seen_list.append(line)
+        #             seen_list = list(set(seen_list))
+        #             w_txt = "\n".join(seen_list)
+        #             f_w.write(w_txt + "\n")
+        #         os.remove(shard_path)  # 删除临时文件
+        #
+        # with open(result_w_path, "r", encoding="utf-8") as fp_r:
+        #     line_count = sum(1 for _ in fp_r)
+        # self.__logger.info(f"文件处理完毕,去重后总行数为：{line_count},结果路径：{result_w_path}")

{xtn_tools_pro-1.0.0.7.1.dist-info → xtn_tools_pro-1.0.0.7.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.7.1
+Version: 1.0.0.7.3
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@gmail.com
@@ -15,5 +15,7 @@ Requires-Dist: dbutils
 Requires-Dist: colorlog
 Requires-Dist: requests
 Requires-Dist: Faker
+Requires-Dist: PyJWT
+Requires-Dist: tqdm
 
 xtnkk-tools

{xtn_tools_pro-1.0.0.7.1.dist-info → xtn_tools_pro-1.0.0.7.3.dist-info}/RECORD

@@ -12,17 +12,18 @@ xtn_tools_pro/proxy/proxy.py,sha256=No6E1pFY5yx2F4976pXPrLtq-QEVp79KupzcufjSN58,
 xtn_tools_pro/task_pro/__init__.py,sha256=nK3U47hWwE1H875ieEToH9r-jzXHS-PXk8cDstOvRE8,418
 xtn_tools_pro/task_pro/go_fun.py,sha256=hWEt2uJ9FCvJH7PhVZttS-11A7J6zbRKwX7c5YLYQag,19144
 xtn_tools_pro/task_pro/go_fun_v2.py,sha256=SgcXgtEBGSVL1V2LyqO0z8Md2H8JZxucYrLLIwqtiLM,18489
-xtn_tools_pro/task_pro/go_fun_v3.py,sha256=wPd7LFQlDhLH-5SIlq_1k8og5oW6KWeY6g3QvmZSFSU,16013
+xtn_tools_pro/task_pro/go_fun_v3.py,sha256=J8jJLRckbbk4AaHsxUSQ_NKTCIcf71hcPFTDKtJzYD4,16168
 xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
-xtn_tools_pro/utils/crypto.py,sha256=RZ5AET4udlraACWMeNF-17JiZ2R6Ahb47_j4tjkV7LE,3190
-xtn_tools_pro/utils/file_utils.py,sha256=VfdIxog4s1UW5NpKkCvQsUs9qHjLoNCnstZbnftkT4w,2046
+xtn_tools_pro/utils/crypto.py,sha256=oyzFqWum_oimUtzhfVCELQhdMjxDbLu-nOWfcNmazcc,4087
+xtn_tools_pro/utils/file_utils.py,sha256=obaBP7CaBCsXxzqGeWzV2l0yw7vicgKOaXzmpMV8ips,2567
 xtn_tools_pro/utils/helpers.py,sha256=H-a3gnahIah3kJqyKzzKlPWtVQYcFlJncz2rAfBqIiw,4444
-xtn_tools_pro/utils/log.py,sha256=m0WtTWkkwtrki1ftP8vCDR8bMfK2gcfUGx5J2x2IlLQ,10138
+xtn_tools_pro/utils/log.py,sha256=pAye_sXH-y-8v2vNf-OwOTk2Exkjl6y7V_y_Hpk_d0s,10176
 xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
+xtn_tools_pro/utils/set_data.py,sha256=vNhE_jCG-3p6KFnY_jbQ0vQ7EV1gB9D4Jb0S5ZoD4IM,7529
 xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
 xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
-xtn_tools_pro-1.0.0.7.1.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.7.1.dist-info/METADATA,sha256=Yl6AU8kAvrr4nKEBgCHpLCJbOXtZ7UksCEF7XU-Gtuo,455
-xtn_tools_pro-1.0.0.7.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.7.1.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.7.1.dist-info/RECORD,,
+xtn_tools_pro-1.0.0.7.3.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.7.3.dist-info/METADATA,sha256=e7BnO3AKRCicejlwWdIVafht9964e0EJmA5ORQeRPa0,498
+xtn_tools_pro-1.0.0.7.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.7.3.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.7.3.dist-info/RECORD,,