xtn-tools-pro 1.0.0.7.2__py3-none-any.whl → 1.0.0.7.4__py3-none-any.whl

xtn_tools_pro/task_pro/go_fun_v3.py

@@ -34,9 +34,11 @@ class GoFunTaskV3:
         restart_time = ini_dict.get('restart_time', 0)  # 间隔x秒强制重启
         restart_time = 30 * 60 if restart_time <= 0 else restart_time  # 间隔x秒强制重启时间不传默认60分钟
         update_proxies_time = ini_dict.get('update_proxies_time', 0)  # 间隔x秒更新代理
-        upload_task_tine = ini_dict.get('upload_task_tine', 0)  # 回写间隔
-        download_not_task_tine = ini_dict.get('download_not_task_tine', 0)  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
-        download_not_task_tine = 2 if download_not_task_tine <= 0 else download_not_task_tine  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        upload_task_time = ini_dict.get('upload_task_time', 0)  # 回写间隔
+        download_not_task_time = ini_dict.get('download_not_task_time', 0)  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+        download_not_task_time = 2 if download_not_task_time <= 0 else download_not_task_time  # 当遇到下载任务接口返回空任务时,间隔x秒再继续请求,默认2秒
+
+        # 默认进程数和线程数
         processes_num = 1 if processes_num <= 0 else processes_num
         thread_num = 1 if thread_num <= 0 else thread_num
 
@@ -62,18 +64,18 @@ class GoFunTaskV3:
             "thread_num": int(thread_num),
             "restart_time": int(restart_time),
             "update_proxies_time": int(update_proxies_time),
-            "upload_task_tine": int(upload_task_tine),
+            "upload_task_time": int(upload_task_time),
             "download_url": download_url,  # 获取任务地址
             "upload_url": upload_url,  # 回写任务地址
             "update_proxy_url": update_proxy_url,  # 更新代理地址
             "external_ip": external_ip,
-            "download_not_task_tine": download_not_task_tine,
+            "download_not_task_time": download_not_task_time,
         }
 
         logger.debug(
             f"\n无敌框架来咯~~~当前设置配置如下:"
             f"\n\t功能函数重启间隔:{restart_time};进程数:{processes_num};线程数:{thread_num}"
-            f"\n\t代理更新间隔:{update_proxies_time};回写间隔{upload_task_tine};\n"
+            f"\n\t代理更新间隔:{update_proxies_time};回写间隔{upload_task_time};\n"
         )
 
         # 共享任务队列
@@ -173,6 +175,7 @@ class GoFunTaskV3:
         download_url = ini_info["download_url"]
         auto = ini_info["auto"]
         task = ini_info["task"]
+        download_not_task_time = ini_info["download_not_task_time"]
         headers = {"Authorization": auto}
         params = {"taskType": task}
 
@@ -182,7 +185,7 @@ class GoFunTaskV3:
                 qsize = download_queue.qsize()
                 logger.info(f"当前队列剩余任务数:{qsize}")
                 if qsize >= 10:
-                    time.sleep(2)
+                    time.sleep(download_not_task_time)
                     continue
                 resp = requests.get(download_url, headers=headers, params=params, timeout=5)
                 json_data = resp.json()
@@ -218,7 +221,7 @@ class GoFunTaskV3:
         external_ip = ini_info["external_ip"]
         auto = ini_info["auto"]
         task = ini_info["task"]
-        upload_task_tine = ini_info["upload_task_tine"]
+        upload_task_time = ini_info["upload_task_time"]
         headers = {"Authorization": auto}
         params = {"taskType": task}
         while True:
@@ -245,7 +248,8 @@ class GoFunTaskV3:
                     }
                     result_list.append(task_item)
             except Exception as e:
-                logger.critical(f"循环全部获取队列的任务{e}")
+                pass
+                # logger.critical(f"循环全部获取队列的任务{e}")
 
             # 回写任务
             data = {"result": result_list, "remoteAddr": external_ip}
@@ -260,10 +264,10 @@ class GoFunTaskV3:
                     logger.critical(f"回写异常,{len(result_list)},{e}")
                     time.sleep(2)
 
-            if not upload_task_tine:
+            if not upload_task_time:
                 # 一直执行 不退出
                 continue
-            time.sleep(upload_task_tine)
+            time.sleep(upload_task_time)
 
     def __update_proxy(self, proxies_dict, manager_info, ini_info, logger):
         """

xtn_tools_pro/utils/file_utils.py

@@ -58,6 +58,28 @@ def check_file_exists(file_path):
     return False
 
 
+def is_dir(dir_path):
+    """
+        传入文件夹判断是否为文件夹或文件夹是否存在
+        传入的如果是文件路径则返回False
+        传入的如果是一个不存在的文件夹则返回False
+    :param dir_path:
+    :return:
+    """
+    if not os.path.isdir(dir_path):
+        return False
+    return True
+
+
+def get_listdir(dir_path):
+    """
+        获取指定文件夹下的所有文件
+    :param dir_path:
+    :return:
+    """
+    return os.listdir(dir_path)
+
+
 if __name__ == '__main__':
     pass
     print(get_file_extension('file/2024-04-19/BOSCH GEX 125-1A/125-1AE砂磨机操作说明书:[1]_jingyan.txt'))

xtn_tools_pro/utils/log.py

@@ -78,7 +78,7 @@ class Log:
                  is_write_to_file=False,
                  color=True,
                  mode='a',
-                 max_bytes=1024000000,
+                 max_bytes=1024000000,  # 1 * 1024 * 1024 * 1024
                  backup_count=0,
                  encoding="utf-8",
                  save_time_log_path=None):
@@ -107,7 +107,8 @@ class Log:
             # 记录日志文件的路径模板，用于之后创建新的日志文件
             self.path_template = self.path_template = f"{save_time_log_path}/{{date}}/{name}.log"
             path = self.path_template.format(date=self.current_date)
-            mkdirs_dir(self.path_template.format(date=self.current_date))
+            if is_write_to_file:
+                mkdirs_dir(self.path_template.format(date=self.current_date))
 
         # 创建日志格式化器
         # formatter = logging.Formatter('[%(now_datestr)s] [%(levelname)s] [%(func_name)s] - %(message)s') # 原

xtn_tools_pro/utils/set_data.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明
+#    大文件去重
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2025/3/11    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import os
+import fnmatch
+import hashlib
+from tqdm import tqdm
+from xtn_tools_pro.utils.log import Log
+from xtn_tools_pro.utils.helpers import get_orderId_random
+from xtn_tools_pro.utils.file_utils import mkdirs_dir, get_file_extension, is_dir, get_listdir
+
+
+class PSetDataObj:
+    def __init__(self):
+        # 随机生成一个临时文件夹
+        self.__order_id = get_orderId_random()
+        temp_dir_name = f"temp_{self.__order_id}\\"
+        now_current_working_dir = os.getcwd()
+        self.__now_current_working_dir = os.path.join(now_current_working_dir, temp_dir_name)
+        mkdirs_dir(self.__now_current_working_dir)
+
+        self.__logger = Log('set_data', './xxx.log', log_level='DEBUG', is_write_to_console=True,
+                            is_write_to_file=False,
+                            color=True, mode='a', save_time_log_path='./logs')
+
+    def set_file_data_air(self, set_file_path, num_shards=1000):
+        """
+            对单个文件去重，air版本，不对文件做任何修改，去重任何数据
+        :param set_file_path:单文件路径
+        :param num_shards:临时文件切片，推荐:数据越大值越大 1000
+        :return:
+        """
+        if get_file_extension(set_file_path) != ".txt":
+            self.__logger.critical("文件不合法，只接受.txt文件")
+            return
+        self.__logger.info("正在读取文件总行数...")
+
+        with open(set_file_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"读取文件完成,总行数为:{line_count}")
+
+        num_shards = 3000 if num_shards >= 3000 else num_shards
+        num_shards = 3000 if line_count >= 30000000 else num_shards
+        num_shards = 1000 if num_shards <= 1 else num_shards
+
+        shard_file_obj_list = []
+        shard_path_list = []
+        for _ in range(num_shards):
+            shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{_}.tmp')}"
+            shard_path_list.append(shard_path)
+            shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))
+
+        with open(set_file_path, "r", encoding="utf-8") as f_r:
+            tqdm_f = tqdm(f_r, total=line_count, desc="正在去重(1/2)",
+                          bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+            for line_i in tqdm_f:
+                line = line_i.strip().encode()
+                line_hash = hashlib.md5(line).hexdigest()
+                shard_id = int(line_hash, 16) % num_shards
+                shard_file_obj_list[shard_id].write(line_i)
+
+        for shard_file_obj in shard_file_obj_list:
+            shard_file_obj.close()
+
+        result_w_path = os.path.join(self.__now_current_working_dir, "000_去重结果.txt")
+        tqdm_f = tqdm(shard_path_list, total=len(shard_path_list), desc="正在去重(2/2)",
+                      bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+        with open(result_w_path, "w", encoding="utf-8") as f_w:
+            for shard_path in tqdm_f:
+                with open(shard_path, "r", encoding="utf-8") as f_r:
+                    seen_list = []
+                    for line_i in f_r.readlines():
+                        line = line_i.strip()
+                        seen_list.append(line)
+                    seen_list = list(set(seen_list))
+                    w_txt = "\n".join(seen_list)
+                    f_w.write(w_txt + "\n")
+                os.remove(shard_path)  # 删除临时文件
+
+        with open(result_w_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")
+
+    def set_file_data_pro(self, set_file_dir_path, num_shards=3000):
+        """
+            对文件夹下的所有txt文件去重，pro版本，不对文件做任何修改，去重任何数据
+        :param set_file_dir_path:文件夹路径
+        :param num_shards:临时文件切片，推荐:数据越大值越大 1000
+        :return:
+        """
+        if not is_dir(set_file_dir_path):
+            self.__logger.critical("文件夹不存在或不合法")
+            return
+
+        self.__logger.info("正在统计文件可去重数量...")
+        set_file_path_list = []
+        for set_file_name in get_listdir(set_file_dir_path):
+            if fnmatch.fnmatch(set_file_name, '*.txt'):
+                set_file_path_list.append(os.path.join(set_file_dir_path, set_file_name))
+        self.__logger.info(f"当前文件夹下可去重文件数量为:{len(set_file_path_list)}")
+
+        num_shards = 3000 if num_shards >= 3000 else num_shards
+        num_shards = 1000 if num_shards <= 1000 else num_shards
+
+        shard_file_obj_list = []
+        shard_path_list = []
+        for _ in range(num_shards):
+            shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{_}.tmp')}"
+            shard_path_list.append(shard_path)
+            shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))
+
+        for _ in range(len(set_file_path_list)):
+            set_file_path = set_file_path_list[_]
+            with open(set_file_path, "r", encoding="utf-8") as fp_r:
+                line_count = sum(1 for _ in fp_r)
+            # self.__logger.info(f"{set_file_path}读取完成,总行数为:{line_count}")
+
+            with open(set_file_path, "r", encoding="utf-8") as f_r:
+                tqdm_f = tqdm(f_r, total=line_count, desc=f"正在去重({_ + 1}/{len(set_file_path_list) + 1})",
+                              bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+                for line_i in tqdm_f:
+                    line = line_i.strip().encode()
+                    line_hash = hashlib.md5(line).hexdigest()
+                    shard_id = int(line_hash, 16) % num_shards
+                    shard_file_obj_list[shard_id].write(line_i)
+
+        for shard_file_obj in shard_file_obj_list:
+            shard_file_obj.close()
+
+        result_w_path = os.path.join(self.__now_current_working_dir, "000_去重结果.txt")
+        tqdm_f = tqdm(shard_path_list, total=len(shard_path_list),
+                      desc=f"正在去重({len(set_file_path_list) + 1}/{len(set_file_path_list) + 1})",
+                      bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+        with open(result_w_path, "w", encoding="utf-8") as f_w:
+            for shard_path in tqdm_f:
+                with open(shard_path, "r", encoding="utf-8") as f_r:
+                    seen_list = []
+                    for line_i in f_r.readlines():
+                        line = line_i.strip()
+                        seen_list.append(line)
+                    seen_list = list(set(seen_list))
+                    w_txt = "\n".join(seen_list)
+                    f_w.write(w_txt + "\n")
+                os.remove(shard_path)  # 删除临时文件
+
+        with open(result_w_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")
+
+    def merging_data(self, file_dir_path, merging_new_file_name="合并"):
+        """
+            传入一个文件夹,合并这个文件夹下所有.txt的数据
+        :param file_dir_path: 文件夹
+        :param merging_new_file_name: 新的输出位置
+        :return:
+        """
+        if not is_dir(file_dir_path):
+            self.__logger.critical("文件夹不存在或不合法")
+            return
+
+        self.__logger.info("正在统计文件可合并数量...")
+        file_path_list = []
+        for set_file_name in get_listdir(file_dir_path):
+            if fnmatch.fnmatch(set_file_name, '*.txt'):
+                if set_file_name == f"{merging_new_file_name}.txt": continue
+                file_path_list.append(os.path.join(file_dir_path, set_file_name))
+        self.__logger.info(f"当前文件夹下可合并文件数量为:{len(file_path_list)}")
+
+        result_w_path = os.path.join(file_dir_path, f"{merging_new_file_name}.txt")
+
+        with open(result_w_path, "w", encoding="utf-8") as f_w:
+            for _ in range(len(file_path_list)):
+                file_path = file_path_list[_]
+                with open(file_path, "r", encoding="utf-8") as fp_r:
+                    line_count = sum(1 for _ in fp_r)
+
+                with open(file_path, "r", encoding="utf-8") as f_r:
+                    tqdm_f = tqdm(f_r, total=line_count,
+                                  desc=f"正在合并({_ + 1}/{len(file_path_list)})",
+                                  bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+                    for line_i in tqdm_f:
+                        line = line_i.strip()
+                        f_w.write(line + "\n")
+
+    def split_data(self, file_path, split_new_file_name="分割", file_index=1, file_max_line=1000000):
+        if get_file_extension(file_path) != ".txt":
+            self.__logger.critical("文件不合法，只接受.txt文件")
+            return
+        self.__logger.info("正在读取文件总行数...")
+
+        with open(file_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"读取文件完成,总行数为:{line_count}")
+
+        with open(file_path, "r", encoding="utf-8") as f_r:
+            tqdm_f = tqdm(f_r, total=line_count, desc="正在分割(1/1)",
+                          bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+            temp_line_list = []
+            parent_path = os.path.dirname(file_path)
+            for line_i in tqdm_f:
+                line = line_i.strip()
+                temp_line_list.append(line)
+                if len(temp_line_list) == file_max_line:
+                    result_w_path = os.path.join(parent_path, f"{split_new_file_name}_{file_index}.txt")
+                    self.__list_to_write_file(result_w_path, temp_line_list)
+                    temp_line_list = []
+                    file_index += 1
+            if temp_line_list:
+                result_w_path = os.path.join(parent_path, f"{split_new_file_name}_{file_index}.txt")
+                self.__list_to_write_file(result_w_path, temp_line_list)
+
+    def __list_to_write_file(self, file_w_path, data_list):
+        with open(file_w_path, "w", encoding="utf-8") as result_w_f:
+            result_w_f.write("\n".join(data_list))

{xtn_tools_pro-1.0.0.7.2.dist-info → xtn_tools_pro-1.0.0.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.7.2
+Version: 1.0.0.7.4
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@gmail.com
@@ -16,5 +16,6 @@ Requires-Dist: colorlog
 Requires-Dist: requests
 Requires-Dist: Faker
 Requires-Dist: PyJWT
+Requires-Dist: tqdm
 
 xtnkk-tools

{xtn_tools_pro-1.0.0.7.2.dist-info → xtn_tools_pro-1.0.0.7.4.dist-info}/RECORD

@@ -12,17 +12,18 @@ xtn_tools_pro/proxy/proxy.py,sha256=No6E1pFY5yx2F4976pXPrLtq-QEVp79KupzcufjSN58,
 xtn_tools_pro/task_pro/__init__.py,sha256=nK3U47hWwE1H875ieEToH9r-jzXHS-PXk8cDstOvRE8,418
 xtn_tools_pro/task_pro/go_fun.py,sha256=hWEt2uJ9FCvJH7PhVZttS-11A7J6zbRKwX7c5YLYQag,19144
 xtn_tools_pro/task_pro/go_fun_v2.py,sha256=SgcXgtEBGSVL1V2LyqO0z8Md2H8JZxucYrLLIwqtiLM,18489
-xtn_tools_pro/task_pro/go_fun_v3.py,sha256=wPd7LFQlDhLH-5SIlq_1k8og5oW6KWeY6g3QvmZSFSU,16013
+xtn_tools_pro/task_pro/go_fun_v3.py,sha256=J8jJLRckbbk4AaHsxUSQ_NKTCIcf71hcPFTDKtJzYD4,16168
 xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
 xtn_tools_pro/utils/crypto.py,sha256=oyzFqWum_oimUtzhfVCELQhdMjxDbLu-nOWfcNmazcc,4087
-xtn_tools_pro/utils/file_utils.py,sha256=VfdIxog4s1UW5NpKkCvQsUs9qHjLoNCnstZbnftkT4w,2046
+xtn_tools_pro/utils/file_utils.py,sha256=obaBP7CaBCsXxzqGeWzV2l0yw7vicgKOaXzmpMV8ips,2567
 xtn_tools_pro/utils/helpers.py,sha256=H-a3gnahIah3kJqyKzzKlPWtVQYcFlJncz2rAfBqIiw,4444
-xtn_tools_pro/utils/log.py,sha256=m0WtTWkkwtrki1ftP8vCDR8bMfK2gcfUGx5J2x2IlLQ,10138
+xtn_tools_pro/utils/log.py,sha256=mf5huJDA8xVxxFWPG_tl_vOsAA2_ywGDFycYSGHIDCo,10202
 xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
+xtn_tools_pro/utils/set_data.py,sha256=IthfAclck7AbaxOIKOgJZ2wdcfEmlvC-C63Tywcr4bA,11180
 xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
 xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
-xtn_tools_pro-1.0.0.7.2.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.7.2.dist-info/METADATA,sha256=3yoxlnRP7RmdnMWzkHg8OpenKAmOWzzHAZAkLXbKKsk,477
-xtn_tools_pro-1.0.0.7.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.7.2.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.7.2.dist-info/RECORD,,
+xtn_tools_pro-1.0.0.7.4.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.7.4.dist-info/METADATA,sha256=gwssgHhEwWGdiJBfIaTSKm6fDbZYU1AqPYJjTFm8EtE,498
+xtn_tools_pro-1.0.0.7.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.7.4.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.7.4.dist-info/RECORD,,