xtn-tools-pro 1.0.1.0.4__py3-none-any.whl → 1.0.1.0.5__py3-none-any.whl

xtn_tools_pro/utils/set_data.py

@@ -30,15 +30,28 @@ class PSetDataObj:
                             is_write_to_file=False,
                             color=True, mode='a', save_time_log_path='./logs')
 
+    def get_file_line_cnt(self, txt_file_path):
+        """
+            传入一个 txt 文件 读取总行数
+        :param txt_file_path:
+        :return:
+        """
+        if get_file_extension(txt_file_path) != ".txt":
+            return 0
+
+        with open(txt_file_path, "r", encoding="utf-8") as fp_r:
+            a_line_count = sum(1 for _ in fp_r)
+        return a_line_count
+
     def set_file_data_air(self, set_file_path, num_shards=1000):
         """
-            对单个文件去重，air版本，不对文件做任何修改，去重任何数据
+            对单个文件去重,air版本,不对文件做任何修改,去重任何数据
         :param set_file_path:单文件路径
-        :param num_shards:临时文件切片，推荐:数据越大值越大 1000
+        :param num_shards:临时文件切片,推荐:数据越大值越大 1000
         :return:
         """
         if get_file_extension(set_file_path) != ".txt":
-            self.__logger.critical("文件不合法，只接受.txt文件")
+            self.__logger.critical("文件不合法,只接受.txt文件")
             return
         self.__logger.info("正在读取文件总行数...")
 
@@ -90,9 +103,9 @@ class PSetDataObj:
 
     def set_file_data_pro(self, set_file_dir_path, num_shards=3000):
         """
-            对文件夹下的所有txt文件去重，pro版本，不对文件做任何修改，去重任何数据
+            对文件夹下的所有txt文件去重,pro版本,不对文件做任何修改,去重任何数据
         :param set_file_dir_path:文件夹路径
-        :param num_shards:临时文件切片，推荐:数据越大值越大 1000
+        :param num_shards:临时文件切片,推荐:数据越大值越大 1000
         :return:
         """
         if not is_dir(set_file_dir_path):
@@ -154,6 +167,111 @@ class PSetDataObj:
             line_count = sum(1 for _ in fp_r)
         self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")
 
+    def set_file_data_max(self, a_set_file_path, b_set_file_path):
+        """
+            对两个a、b文件去重,从a文件的元素中剔除掉b文件里所有元素
+        :param a_set_file_path: a文件路径
+        :param b_set_file_path: b文件路径
+        :return: 
+        """
+        if get_file_extension(a_set_file_path) != ".txt" or get_file_extension(b_set_file_path) != ".txt":
+            self.__logger.critical("文件不合法,只接受.txt文件")
+            return
+        self.__logger.info("正在读取a、b文件总行数...")
+        a_line_count = self.get_file_line_cnt(a_set_file_path)
+        self.__logger.info(f"读取a文件完成,总行数为:{a_line_count},路径:{a_set_file_path}")
+        b_line_count = self.get_file_line_cnt(b_set_file_path)
+        self.__logger.info(f"读取b文件完成,总行数为:{b_line_count},路径:{b_set_file_path}")
+
+        num_shards = 50000
+        buffer_size = 2500  # 缓冲区行数，越大I/O越少但内存占用越高
+        # self.__now_current_working_dir = r"D:\000\xtnkk-tools\demos\temp_25032714561875181921"
+        # self.__order_id = "25032714561875181921"
+
+        # 初始化文件
+        ab_shard_path_dict = {}
+        for _ in range(num_shards):
+            a_shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{_}_a.tmp')}"
+            b_shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{_}_b.tmp')}"
+            ab_shard_path_dict[_] = {
+                "a": a_shard_path,
+                "b": b_shard_path,
+            }
+            with open(a_shard_path, "w", encoding="utf-8"):
+                pass
+            with open(b_shard_path, "w", encoding="utf-8"):
+                pass
+
+        self.__set_file_data_max_writelinesBuffers(a_set_file_path, a_line_count, num_shards, buffer_size, "a")
+        self.__set_file_data_max_writelinesBuffers(b_set_file_path, b_line_count, num_shards, buffer_size, "b")
+
+        result_w_path = os.path.join(self.__now_current_working_dir, "000_去重结果.txt")
+        result_tqdm_f = tqdm(ab_shard_path_dict, total=len(ab_shard_path_dict),
+                             desc=f"正在合并文件", bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
+
+        with open(result_w_path, "w", encoding="utf-8") as result_f_w:
+            for _ in result_tqdm_f:
+                a_shard_path = ab_shard_path_dict[_]["a"]
+                b_shard_path = ab_shard_path_dict[_]["b"]
+                a_seen_list = []
+                b_seen_list = []
+                with open(a_shard_path, "r", encoding="utf-8") as a_f_r:
+                    for line_i in a_f_r.readlines():
+                        line = line_i.strip()
+                        a_seen_list.append(line)
+                with open(b_shard_path, "r", encoding="utf-8") as b_f_r:
+                    for line_i in b_f_r.readlines():
+                        line = line_i.strip()
+                        b_seen_list.append(line)
+
+                seen_list = list(set(a_seen_list) - set(b_seen_list))
+                w_txt = "\n".join(seen_list)
+                result_f_w.write(w_txt + "\n")
+                os.remove(a_shard_path)  # 删除临时文件
+                os.remove(b_shard_path)  # 删除临时文件
+
+        with open(result_w_path, "r", encoding="utf-8") as fp_r:
+            line_count = sum(1 for _ in fp_r)
+        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")
+
+    def __set_file_data_max_writelinesBuffers(self, set_file_path, file_line_count, num_shards, buffer_size,
+                                              tmp_file_suffix):
+        """
+            set_file_data_max 专用,用于写临时文件
+        :param set_file_path:
+        :param file_line_count:
+        :param num_shards:
+        :param buffer_size:
+        :param tmp_file_suffix:
+        :return:
+        """
+        # 初始化分片缓冲区
+        buffers_dict = {i: [] for i in range(num_shards)}
+        with open(set_file_path, "r", encoding="utf-8") as a_f_r:
+            for line_i in tqdm(a_f_r,
+                               total=file_line_count,
+                               desc=f"正在去重{tmp_file_suffix}文件",
+                               bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]"):
+                line = line_i.strip().encode()
+                line_hash = hashlib.md5(line).hexdigest()
+                shard_id = int(line_hash, 16) % num_shards
+                buffers_dict[shard_id].append(line_i)  # 写入缓冲区
+
+                # 缓冲满时写入
+                if len(buffers_dict[shard_id]) >= buffer_size:
+                    shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{shard_id}_{tmp_file_suffix}.tmp')}"
+                    with open(shard_path, "a", encoding="utf-8") as a_shard_fw:
+                        a_shard_fw.writelines(buffers_dict[shard_id])
+                        buffers_dict[shard_id].clear()
+
+            # 最终刷新所有缓冲区
+            for shard_id in buffers_dict:
+                if buffers_dict[shard_id]:
+                    a_shard_path = f"{os.path.join(self.__now_current_working_dir, f'{self.__order_id}_shard_{shard_id}_{tmp_file_suffix}.tmp')}"
+                    with open(a_shard_path, "a", encoding="utf-8") as a_shard_fw:
+                        a_shard_fw.writelines(buffers_dict[shard_id])
+                        buffers_dict[shard_id].clear()
+
     def merging_data(self, file_dir_path, merging_new_file_name="合并"):
         """
             传入一个文件夹,合并这个文件夹下所有.txt的数据
@@ -190,8 +308,16 @@ class PSetDataObj:
                         f_w.write(line + "\n")
 
     def split_data(self, file_path, split_new_file_name="分割", file_index=1, file_max_line=1000000):
+        """
+            传入一个txt文件,按 file_max_line 分割
+        :param file_path:
+        :param split_new_file_name:
+        :param file_index:
+        :param file_max_line:
+        :return:
+        """
         if get_file_extension(file_path) != ".txt":
-            self.__logger.critical("文件不合法，只接受.txt文件")
+            self.__logger.critical("文件不合法,只接受.txt文件")
             return
         self.__logger.info("正在读取文件总行数...")
 
@@ -217,5 +343,11 @@ class PSetDataObj:
                 self.__list_to_write_file(result_w_path, temp_line_list)
 
     def __list_to_write_file(self, file_w_path, data_list):
+        """
+            列表数据 批量 覆盖写入文件
+        :param file_w_path:
+        :param data_list:
+        :return:
+        """
         with open(file_w_path, "w", encoding="utf-8") as result_w_f:
             result_w_f.write("\n".join(data_list))

{xtn_tools_pro-1.0.1.0.4.dist-info → xtn_tools_pro-1.0.1.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.1.0.4
+Version: 1.0.1.0.5
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@gmail.com

{xtn_tools_pro-1.0.1.0.4.dist-info → xtn_tools_pro-1.0.1.0.5.dist-info}/RECORD

@@ -58,12 +58,12 @@ xtn_tools_pro/utils/file_utils.py,sha256=obaBP7CaBCsXxzqGeWzV2l0yw7vicgKOaXzmpMV
 xtn_tools_pro/utils/helpers.py,sha256=OmPRxB6vDN1S7OhxQWgoOXnxs2YPorwSQKBVzYOiKoA,4928
 xtn_tools_pro/utils/log.py,sha256=mf5huJDA8xVxxFWPG_tl_vOsAA2_ywGDFycYSGHIDCo,10202
 xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
-xtn_tools_pro/utils/set_data.py,sha256=IthfAclck7AbaxOIKOgJZ2wdcfEmlvC-C63Tywcr4bA,11180
+xtn_tools_pro/utils/set_data.py,sha256=BzCLbEDO83csDHBey8kP7SoE6kx6EjqE7OqRBY7k82s,17608
 xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
 xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
-xtn_tools_pro-1.0.1.0.4.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.1.0.4.dist-info/METADATA,sha256=NXfWI4TvsFCXCr4IZMpsqP-vUP46smejlUEm8sFXFVY,498
-xtn_tools_pro-1.0.1.0.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.1.0.4.dist-info/entry_points.txt,sha256=t8CtXWOgw7nRDW3XNlZh8MT_P4l8EzsFnyWi5b3ovrc,68
-xtn_tools_pro-1.0.1.0.4.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.1.0.4.dist-info/RECORD,,
+xtn_tools_pro-1.0.1.0.5.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.1.0.5.dist-info/METADATA,sha256=8gl2HpIsyu3m2geeB0cjy5a8OqPxDxxOBW6ILMN56eE,498
+xtn_tools_pro-1.0.1.0.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.1.0.5.dist-info/entry_points.txt,sha256=t8CtXWOgw7nRDW3XNlZh8MT_P4l8EzsFnyWi5b3ovrc,68
+xtn_tools_pro-1.0.1.0.5.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.1.0.5.dist-info/RECORD,,