pyxllib 0.3.60.3__tar.gz → 0.3.62__tar.gz

{pyxllib-0.3.60.3/pyxllib.egg-info → pyxllib-0.3.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.60.3
+Version: 0.3.62
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.60.3 → pyxllib-0.3.62}/pyxllib/file/specialist/__init__.py

@@ -4,6 +4,9 @@
 # @Email  : 877362867@qq.com
 # @Date   : 2021/06/06 17:46
 
+from itertools import islice
+import multiprocessing
+import multiprocessing.dummy
 
 from pyxllib.file.specialist.filelib import *
 from pyxllib.file.specialist.dirlib import *
@@ -50,15 +53,45 @@ class JsonlDataFile:
                 # 只读取部分数据
                 self.read_partial_records(num_records)
 
+    def __len__(self):
+        return len(self.records)
+
+    def yield_record(self, start=0, end=None, step=1, batch_size=None):
+        """ 返回指定区间的记录
+
+        :param int start: 起始记录索引，默认为0
+        :param int end: 结束记录索引，默认为None（读取到记录末尾）
+        :param int step: 步长，默认为1
+        :param int batch_size: 每批返回的记录数，如果为None，则逐记录返回
+        """
+        total_records = len(self.records)  # 获取总记录数
+
+        # 处理负索引
+        if start < 0 or (end is not None and end < 0):
+            if start < 0:
+                start = total_records + start
+            if end is not None and end < 0:
+                end = total_records + end
+
+        iterator = islice(self.records, start, end, step)
+        while True:
+            batch = list(islice(iterator, batch_size))
+            if not batch:
+                break
+            if batch_size is None:
+                yield from batch
+            else:
+                yield batch
+
     def read_partial_records(self, num_records):
         """ 从jsonl文件中只读取指定数量的记录 """
         if self.infile and self.infile.is_file():
-            with open(self.infile, 'r', encoding='utf-8') as file:
-                for _ in range(num_records):
-                    line = file.readline().strip()
-                    if not line:
-                        break  # 如果已经读完文件，跳出循环
+            try:
+                lines = next(self.infile.yield_line(batch_size=num_records))
+                for line in lines:
                     self.records.append(json.loads(line))
+            except StopIteration:
+                self.records = []
 
     def save(self, outfile=None, ensure_ascii=False):
         """ 将当前数据保存到指定的jsonl文件中 """
@@ -208,31 +241,24 @@ class JsonlDataFile:
         self.records += other.records
         return self
 
-    def apply_function_to_records(self, func, inplace=False, print_mode=0):
+    def process_each_record(self, func, *, inplace=False, print_mode=0, threads_num=1):
         """ 对records中的每个record应用函数func，可以选择是否在原地修改，以及是否显示进度条
 
         :param function func: 对record进行处理的函数，应接受一个record作为参数并返回处理后的record，如果返回None则删除该record
         :param bool inplace: 是否在原地修改records，如果为False（默认），则创建新的JsonlDataFile并返回
         :param int print_mode: 是否显示处理过程的进度条，0表示不显示（默认），1表示显示
         :return JsonlDataFile or None: 如果inplace为False，则返回新的JsonlDataFile，否则返回None
+        :param int threads_num: 线程数，默认为1，即单线程
 
         遍历self.records，对每个record执行func函数，如果func返回None，则不包含该record到新的records中。
-
-        >>> data_file = JsonlDataFile()
-        >>> data_file.records = [{'a': 1}, {'b': 2}, {'c': 3}]
-        >>> func = lambda x: {k: v * 2 for k, v in x.items()} if 'a' in x else None  # 定义一个只处理含有'a'的record并将其值翻倍的函数
-        >>> new_data_file = data_file.apply_function_to_records(func, print_mode=1)
-        >>> new_data_file.records
-        [{'a': 2}]
-        >>> data_file.records  # 原始的data_file并没有被修改
-        [{'a': 1}, {'b': 2}, {'c': 3}]
         """
-        records = self.records
-        if print_mode == 1:
-            records = tqdm(records)
+        with multiprocessing.dummy.Pool(threads_num) as executor:
+            if print_mode == 1:
+                results = tqdm(executor.imap(func, self.records), total=len(self.records))
+            else:
+                results = executor.imap(func, self.records)
 
-        # new_records = [func(record) for record in records if func(record) is not None]
-        new_records = [func(record) for record in records]
+            new_records = list(results)
 
         if inplace:
             self.records = new_records
@@ -242,8 +268,14 @@ class JsonlDataFile:
             new_data_file.records = new_records
             return new_data_file
 
+    def update_each_record(self, func, print_mode=0):
+        """ 遍历并对原始数据进行更改 """
+        self.process_each_record(func, inplace=True, print_mode=print_mode)
+
 
 class JsonlDataDir:
+    """ 注意这个类开发目标，应该是尽量去模拟JsonDataFile，让下游工作更好衔接统一 """
+
     def __init__(self, root):
         """ 一般用来处理较大的jsonl文件，将其该放到一个目录里，拆分成多个jsonl文件
 
@@ -252,28 +284,140 @@ class JsonlDataDir:
         self.root = XlPath(root)
         self.files = []
         for f in self.root.glob_files('*.jsonl'):
-            if re.match(r'\d+$', f.stem):
+            if re.match(r'_?\d+$', f.stem):  # 目前先用'_?'兼容旧版，但以后应该固定只匹配_\d+
                 self.files.append(f)
 
+    def __bool__(self):
+        if self.root.is_dir() and self.files:
+            return True
+        else:
+            return False
+
+    def count_records(self):
+        total = 0
+        for f in self.files:
+            total += len(JsonlDataFile(f).records)
+        return total
+
     def check(self):
+        """ 检查一些数据状态 """
         print('文件数：', len(self.files))
 
     @classmethod
-    def init_from_file(cls, file, lines_per_file=1000):
+    def init_from_file(cls, file, lines_per_file=10000):
         """ 从一个jsonl文件初始化一个JsonlDataDir对象 """
         file = XlPath(file)
         dst_dir = file.parent / file.stem
-        if not dst_dir.is_dir():
+        if not dst_dir.is_dir() and file.is_file():
             file.split_to_dir(lines_per_file, dst_dir)
         c = cls(dst_dir)
         return c
 
-    def apply_function_to_records(self, func):
-        """ 对records中的每个record应用函数func，先写出最简单的串行版本，后续可以考虑更复杂的并行版本
+    def rearrange(self, lines_per_file=10000):
+        """ 重新整理划分文件
+
+        :param int lines_per_file: 每个文件的行数
+        """
+        output_dir = self.root
+
+        # 使用临时文件名前缀，以便在处理完成后更改为最终的文件名
+        temp_prefix = 'temp_'
+
+        new_file_count = 1
+        new_file = None
+        line_count = 0
+
+        # 计算总行数以确定文件名的前导零数量
+        total_lines = sum(1 for file in self.files for _ in file.open('r', encoding='utf-8'))
+        num_digits = len(str((total_lines + lines_per_file - 1) // lines_per_file))
+
+        for file in self.files:
+            with file.open('r', encoding='utf-8') as f:
+                for line in f:
+                    if line_count == 0:
+                        if new_file is not None:
+                            new_file.close()
+                        new_file_name = f'{temp_prefix}{new_file_count:0{num_digits}d}.jsonl'
+                        new_file_path = output_dir / new_file_name
+                        new_file = new_file_path.open('w', encoding='utf-8')
+                        new_file_count += 1
+
+                    new_file.write(line)
+                    line_count += 1
+
+                    if line_count == lines_per_file:
+                        line_count = 0
+
+        if new_file is not None:
+            new_file.close()
+
+        # 删除旧文件
+        for file in self.files:
+            os.remove(file)
+
+        # 将临时文件名更改为最终的文件名
+        for temp_file in output_dir.glob(f'{temp_prefix}*.jsonl'):
+            final_name = temp_file.name[len(temp_prefix):]
+            temp_file.rename(output_dir / final_name)
+
+    def yield_record(self, batch_size=None):
+        """ 返回数据记录
+
+        :param int batch_size: 每批返回的记录数，如果为None，则逐条返回
         """
-        n = len(self.files)
         for i, file in enumerate(self.files):
-            print(f'处理文件 {i + 1}/{n}: {file}')
+            data = file.read_jsonl()
+            iterator = iter(data)
+            while True:
+                batch = list(islice(iterator, batch_size))
+                if not batch:
+                    break
+                if batch_size is None:
+                    yield from batch
+                else:
+                    yield batch
+
+    def process_each_record(self, func, *, inplace=False,
+                            print_mode=1, desc=None,
+                            processes_num=1, threads_num=1):
+        """ 封装的对每个record进行操作的函数
+
+        :param int processes_num: 进程数，每个文件为单独一个进程
+        :param int threads_num: 线程数，每个文件处理的时候使用几个线程
+        """
+        for i, file in tqdm(enumerate(self.files), desc=desc, disable=not print_mode):
             data_file = JsonlDataFile(file)
-            data_file.apply_function_to_records(func, inplace=True, print_mode=1)
-            data_file.save(file)
+            data_file.process_each_record(func, inplace=inplace,
+                                          threads_num=threads_num,
+                                          print_mode=print_mode > 1)
+            if inplace:
+                data_file.save(file)
+
+    def update_each_record(self, func, desc=None):
+        """ 封装的对每个record进行操作的函数
+        """
+        self.process_each_record(func, inplace=True, desc=desc)
+
+    def process_each_file(self, func, *, desc=None):
+        for i, file in tqdm(enumerate(self.files), desc=desc):
+            func(file)
+
+    def process_each_records(self, func, *, inplace=False, desc=None):
+        for i, file in tqdm(enumerate(self.files), desc=desc):
+            records = XlPath(file).read_jsonl()
+            new_records = func(records)  # 如果使用inplace，那么需要函数配套返回新的records
+            if inplace:
+                XlPath(file).write_jsonl(new_records)
+
+    def save(self, dst_path=None):
+        """ 将数据合并到一个jsonl文件中 """
+        if not dst_path:
+            dst_path = self.root.parent / f'{self.root.name}.jsonl'
+        dst_path = XlPath(dst_path)
+        dst_path.parent.mkdir(parents=True, exist_ok=True)
+        with dst_path.open('w', encoding='utf8') as f:
+            for file in tqdm(self.files, desc=f'合并文件并保存 {dst_path.name}'):
+                with file.open('r', encoding='utf8') as f2:
+                    for line in f2:
+                        if line.strip():  # 不存储空行
+                            f.write(line)

{pyxllib-0.3.60.3 → pyxllib-0.3.62}/pyxllib/file/specialist/filelib.py

@@ -21,6 +21,8 @@ import tempfile
 import ujson
 from collections import defaultdict, Counter
 import math
+from itertools import islice
+import datetime
 
 # import chardet
 import charset_normalizer
@@ -916,22 +918,57 @@ class XlPath(type(pathlib.Path())):
             # 判断路径字符串是否包含相对路径字符串
             return item_str.startswith(abs_path_str) or abs_path_str == item_str
 
-    def get_line_count(self):
-        """ 统计文件的行数 """
-        with open(self, 'rb') as file:
-            line_count = 0
+    def get_total_lines(self, encoding='utf-8', skip_blank=False):
+        """ 统计文件的行数（注意会统计空行，所以在某些场合可能与预期理解的条目数不太一致）
+
+        :param str encoding: 文件编码，默认为'utf-8'
+        :param bool skip_blank: 是否跳过空白行，默认为True
+        :return: 文件的行数
+        """
+        line_count = 0
+        with open(self, 'r', encoding=encoding) as file:
+            for line in file:
+                if skip_blank and not line.strip():  # 跳过空白行
+                    continue
+                line_count += 1
+        return line_count
+
+    def yield_line(self, start=0, end=None, step=1, batch_size=None, encoding='utf-8'):
+        """ 返回指定区间的文件行
+
+        :param int start: 起始行，默认为0
+        :param int end: 结束行，默认为None（读取到文件末尾）
+        :param int step: 步长，默认为1
+        :param int batch_size: 每批返回的行数，如果为None，则逐行返回
+        """
+        total_lines = None  # 使用局部变量缓存总行数
+        # 处理负索引
+        if start < 0 or (end is not None and end < 0):
+            total_lines = total_lines or self.get_total_lines()
+            if start < 0:
+                start = total_lines + start
+            if end is not None and end < 0:
+                end = total_lines + end
+
+        with open(self, 'r', encoding=encoding) as file:
+            iterator = islice(file, start, end, step)
             while True:
-                chunk = file.read(10 * 1024 * 1024)  # 读取 10MB 的数据块
-                if not chunk:
+                batch = list(islice(iterator, batch_size))
+                if not batch:
                     break
-                line_count += chunk.count(b'\n')  # 计算换行符数量
-        return line_count + 1  # 添加最后一行
+                batch = [line.rstrip('\n') for line in batch]  # 删除每行末尾的换行符
+                if batch_size is None:
+                    yield from batch
+                else:
+                    yield batch
 
-    def split_to_dir(self, lines_per_file, dst_dir=None, encoding='utf-8'):
+    def split_to_dir(self, lines_per_file, dst_dir=None, encoding='utf-8',
+                     filename_template="_{index}{suffix}"):
         """ 将文件按行拆分到多个子文件中
 
         :param int lines_per_file: 打算拆分的每个新文件的行数
-        :param str dst_dir: 目标目录，未输入的时候，输出到同stem明的目录下
+        :param str dst_dir: 目标目录，未输入的时候，输出到同stem名的目录下
+        :param str filename_template: 文件名模板，可以包含 {stem}, {index} 和 {suffix} 占位符
         :return list: 拆分的文件路径列表
             拆分后文件名类似如下： 01.jsonl, 02.jsonl, ...
         """
@@ -950,7 +987,6 @@ class XlPath(type(pathlib.Path())):
 
         # 2 拆分文件
         split_files = []  # 用于保存拆分的文件路径
-        tmp_files = []  # 用于保存临时文件路径
         outfile = None
         filename_format = "{:04d}"
         outfile_index = 0
@@ -964,7 +1000,6 @@ class XlPath(type(pathlib.Path())):
                         outfile.close()
                     outfile_path = dst_dir / f"{self.stem}_{filename_format.format(outfile_index)}{suffix}"
                     outfile = open(outfile_path, 'w', encoding='utf-8')
-                    tmp_files.append(str(outfile_path))
                     split_files.append(outfile_path)  # 先占位，后面再填充
                     outfile_index += 1
                 outfile.write(line)
@@ -976,13 +1011,54 @@ class XlPath(type(pathlib.Path())):
         # 3 重新设置文件名的对齐宽度
         new_filename_format = "{:0" + str(len(str(len(split_files)))) + "d}"
         for i, old_file in enumerate(split_files):
-            new_name = dst_dir / f"{new_filename_format.format(i)}{suffix}"
+            new_name = dst_dir / filename_template.format(stem=self.stem,
+                                                          index=new_filename_format.format(i),
+                                                          suffix=suffix)
             os.rename(old_file, new_name)
             split_files[i] = new_name
 
         # 返回拆分的文件路径列表
         return split_files
 
+    def merge_from_files(self, files,
+                         ignore_empty_lines_between_files=False,
+                         encoding='utf-8'):
+        """ 将多个文件合并到一个文件中
+
+        :param list files: 要合并的文件列表
+        :param bool ignore_empty_lines_between_files: 是否忽略文件间的空行
+        :param str encoding: 文件编码，默认为'utf-8'
+        :return XlPath: 合并后的文件路径
+        """
+        # 合并文件
+        prev_line_end_with_newline = True  # 记录上一次text的最后一个字符是否为'\n'
+        with open(self, 'w', encoding=encoding) as outfile:
+            for i, file in enumerate(files):
+                file = XlPath(file)
+                text = file.read_text(encoding=encoding)
+                if ignore_empty_lines_between_files:
+                    text = text.rstrip('\n')
+                if i > 0 and not prev_line_end_with_newline and text != '':
+                    outfile.write('\n')
+                outfile.write(text)
+                prev_line_end_with_newline = text.endswith('\n')
+
+    def merge_from_dir(self, src_dir, filename_template="_{index}{suffix}", encoding='utf-8'):
+        """ 将目录中的多个文件合并到一个文件中
+
+        :param str src_dir: 要合并的文件所在的目录
+        :param str filename_template: 文件名模板，可以包含 {stem}, {index} 和 {suffix} 占位符
+        :param str encoding: 文件编码，默认为'utf-8'
+        :return XlPath: 合并后的文件路径
+        """
+        src_dir = XlPath(src_dir)
+        stem = src_dir.name
+
+        pattern = filename_template.format(stem=stem, index="(\d+)", suffix=".*")
+        files = [file for file in src_dir.iterdir() if re.match(pattern, file.name)]  # 获取目录中符合模式的文件
+
+        self.merge_from_files(files, ignore_empty_lines_between_files=True, encoding=encoding)
+
     def __1_read_write(self):
         """ 参考标准库的
         read_bytes、read_text
@@ -1089,7 +1165,7 @@ class XlPath(type(pathlib.Path())):
     def write_jsonl(self, list_data, ensure_ascii=False):
         """ 由于这种格式主要是跟商汤这边对接，就尽量跟它们的格式进行兼容 """
         content = '\n'.join([json.dumps(x, ensure_ascii=ensure_ascii) for x in list_data])
-        self.write_text_unix(content + '\n')
+        self.write_text_unix(content)
 
     def read_csv(self, encoding='utf8', *, errors='strict', return_mode: bool = False,
                  delimiter=',', quotechar='"', **kwargs):
@@ -1551,11 +1627,49 @@ class XlPath(type(pathlib.Path())):
         else:
             return msg
 
-    def check_summary(self, print_mode=False, hash_func=None):
+    def check_summary(self, print_mode=True, return_mode=False, **kwargs):
+        if self.is_dir():
+            res = self._check_dir_summary(print_mode, **kwargs)
+        elif self.is_file():
+            res = self._check_file_summary(print_mode, **kwargs)
+        else:
+            res = '文件不存在'
+            print(res)
+
+        if return_mode:
+            return res
+
+    def _check_file_summary(self, print_mode=True, **kwargs):
+        """ 对文件进行通用的状态检查
+
+        :param bool print_mode: 是否将统计信息打印到控制台
+        :return dict: 文件的统计信息
+        """
+        file_summary = {}
+
+        # 文件大小
+        file_summary['文件大小'] = self.size(human_readable=True)
+
+        # 文件行数
+        file_summary['文件行数'] = self.get_total_lines()
+
+        # 文件修改时间
+        mod_time_str = datetime.datetime.fromtimestamp(self.mtime()).strftime('%Y-%m-%d %H:%M:%S')
+        file_summary['修改时间'] = mod_time_str
+
+        # 如果print_mode为True，则将统计信息打印到控制台
+        if print_mode:
+            for key, value in file_summary.items():
+                print(f"{key}: {value}")
+
+        return file_summary
+
+    def _check_dir_summary(self, print_mode=True, hash_func=None, run_mode=99):
         """ 对文件夹情况进行通用的状态检查
 
         :param hash_func: 可以传入自定义的hash函数，用于第四块的重复文件运算
             其实默认的get_etag就没啥问题，只是有时候为了性能考虑，可能会传入一个支持，提前有缓存知道etag的函数
+        :param int run_mode: 只运行编号内的功能
         """
         if not self.is_dir():
             return ''
@@ -1567,43 +1681,48 @@ class XlPath(type(pathlib.Path())):
 
         # 一 目录大小，二 各后缀文件大小
         msg = []
-        printf('【' + self.as_posix() + '】目录检查')
-        printf('\n'.join(self.check_size('list')))
+        if run_mode >= 1:  # 1和2目前是绑定一起运行的
+            printf('【' + self.as_posix() + '】目录检查')
+            printf('\n'.join(self.check_size('list')))
 
         # 三 重名文件
-        printf('\n三、重名文件（忽略大小写，跨目录检查name重复情况）')
-        printf('\n'.join(self.check_repeat_name_files(print_mode=False)))
+        if run_mode >= 3:
+            printf('\n三、重名文件（忽略大小写，跨目录检查name重复情况）')
+            printf('\n'.join(self.check_repeat_name_files(print_mode=False)))
 
         # 四 重复文件
-        printf('\n四、重复文件（etag相同）')
-        printf('\n'.join(self.check_repeat_files(print_mode=False, hash_func=hash_func)))
+        if run_mode >= 4:
+            printf('\n四、重复文件（etag相同）')
+            printf('\n'.join(self.check_repeat_files(print_mode=False, hash_func=hash_func)))
 
         # 五 错误扩展名
-        printf('\n五、错误扩展名')
-        for i, (f1, suffix2) in enumerate(self.xglob_faker_suffix_files('**/*'), start=1):
-            printf(f'{i}、{f1.relpath(self)} -> {suffix2}')
+        if run_mode >= 5:
+            printf('\n五、错误扩展名')
+            for i, (f1, suffix2) in enumerate(self.xglob_faker_suffix_files('**/*'), start=1):
+                printf(f'{i}、{f1.relpath(self)} -> {suffix2}')
 
         # 六 文件配对
-        printf('\n六、文件配对（检查每个目录里stem名称是否配对，列出文件组成不单一的目录结构，请重点检查落单未配对的情况）')
-        prompt = False
-        for root, dirs, files in os.walk(self):
-            suffix_counts = defaultdict(list)
-            for file in files:
-                stem, suffix = os.path.splitext(file)
-                suffix_counts[stem].append(suffix)
-            suffix_counts = {k: tuple(sorted(v)) for k, v in suffix_counts.items()}
-            suffix_counts2 = {v: k for k, v in suffix_counts.items()}  # 反向存储，如果有重复v会进行覆盖
-            ct = Counter(suffix_counts.values())
-            if len(ct.keys()) > 1:
-                printf(root)
-                for k, v in ct.most_common():
-                    tag = f'\t{k}: {v}'
-                    if v == 1:
-                        tag += f'，{suffix_counts2[k]}'
-                    if len(k) > 1 and not prompt:
-                        tag += f'\t标记注解：有{v}组stem相同文件，配套有{k}这些后缀。其他标记同理。'
-                        prompt = True
-                    printf(tag)
+        if run_mode >= 6:
+            printf('\n六、文件配对（检查每个目录里stem名称是否配对，列出文件组成不单一的目录结构，请重点检查落单未配对的情况）')
+            prompt = False
+            for root, dirs, files in os.walk(self):
+                suffix_counts = defaultdict(list)
+                for file in files:
+                    stem, suffix = os.path.splitext(file)
+                    suffix_counts[stem].append(suffix)
+                suffix_counts = {k: tuple(sorted(v)) for k, v in suffix_counts.items()}
+                suffix_counts2 = {v: k for k, v in suffix_counts.items()}  # 反向存储，如果有重复v会进行覆盖
+                ct = Counter(suffix_counts.values())
+                if len(ct.keys()) > 1:
+                    printf(root)
+                    for k, v in ct.most_common():
+                        tag = f'\t{k}: {v}'
+                        if v == 1:
+                            tag += f'，{suffix_counts2[k]}'
+                        if len(k) > 1 and not prompt:
+                            tag += f'\t标记注解：有{v}组stem相同文件，配套有{k}这些后缀。其他标记同理。'
+                            prompt = True
+                        printf(tag)
 
         return '\n'.join(msg)