pyxllib 0.3.100__tar.gz → 0.3.101__tar.gz

{pyxllib-0.3.100/pyxllib.egg-info → pyxllib-0.3.101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.100
+Version: 0.3.101
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.100 → pyxllib-0.3.101}/pyxllib/ext/robustprocfile.py

@@ -13,6 +13,9 @@
 from collections import defaultdict, Counter
 import datetime
 import re
+import json
+
+from tqdm import tqdm
 
 from pyxllib.prog.pupil import check_counter, tprint
 from pyxllib.file.specialist import XlPath, refinepath
@@ -189,3 +192,128 @@ def remove_repeat_base(infile,
     new_len = len(data2)
     print(f'去重后剩余 {new_len}/{src_len} ≈ {new_len / src_len:.2%} 的数据')
     return data2
+
+
+class CacheJsonlFile:
+    """ 流式存储文件 """
+
+    def __init__(self, parent_dir, prefix_stem, tag, batch_size=2000):
+        self.parent_dir = parent_dir
+        self.prefix_stem = prefix_stem
+        self.tag = tag
+        self.cache_text_lines = []
+        self.batch_size = batch_size
+
+        # 缓存时的文件名
+        self.cache_file = XlPath(parent_dir) / f'{prefix_stem}_{tag}.cache.jsonl'
+        self.total = 0
+
+    def append(self, data):
+        for x in data:
+            if isinstance(x, str):
+                self.cache_text_lines.append(x)
+            else:
+                self.cache_text_lines.append(json.dumps(x, ensure_ascii=False))
+        if len(self.cache_text_lines) >= self.batch_size:
+            self.flush()
+
+    def flush(self):
+        """ 刷新，将当前缓存写入文件 """
+        if self.cache_text_lines:
+            if self.total == 0:  # 第一次写入时，删除旧缓存文件
+                self.cache_file.delete()
+
+            self.total += len(self.cache_text_lines)
+            self.parent_dir.mkdir(exist_ok=True, parents=True)
+            with open(self.cache_file, 'a', encoding='utf8') as f:
+                f.write('\n'.join(self.cache_text_lines) + '\n')
+            self.cache_text_lines = []
+
+    def save_all(self):
+        """ 最终存储的文件名 """
+        self.flush()
+        dst_file = self.cache_file.with_stem(f'{self.prefix_stem}_{self.tag}{self.total}_{get_timestamp()}')
+        self.cache_file.rename(dst_file)
+
+
+class CacheJsonlGroupFiles:
+    def __init__(self, parent_dir, prefix_stem, tag, group_func=None, batch_size=2000):
+        self.files = {}
+
+        self.parent_dir = parent_dir
+        self.prefix_stem = prefix_stem
+        self.tag = tag
+        self.group_func = group_func
+        self.batch_size = batch_size
+
+    def append(self, data):
+        groups = defaultdict(list)
+        if self.group_func:
+            for x in data:
+                groups[self.group_func(x)].append(x)
+        else:
+            groups[''] = data
+
+        for k, v in groups.items():
+            if k not in self.files:
+                subtag = f'{k}_{self.tag}' if k not in ('', None) else self.tag
+                self.files[k] = CacheJsonlFile(self.parent_dir, self.prefix_stem, subtag, self.batch_size)
+            self.files[k].append(v)
+
+    def save_all(self):
+        for file in self.files.values():
+            file.save_all()
+
+
+def process_single_file(root,
+                        infile,
+                        tag,
+                        row_func,
+                        *,
+                        cur_idx=1,
+                        total=None,
+                        if_exists='skip',
+                        group_func=None,
+                        batch_size=2000):
+    # 1 缓存路径
+    srcdir = XlPath(root)
+    infile = XlPath.init(infile, root=srcdir)
+    relpath = infile.relative_to(srcdir)
+
+    dstdir = srcdir.parent / f'{srcdir.name}_{tag}' / relpath.parent
+    errdir = srcdir.parent / f'{srcdir.name}_{tag}error' / relpath.parent
+
+    # 2 判断是不是已处理过的文件
+    stem = re.split(r'\d+_\d{14}', relpath.stem)[0]
+    dstfiles = list(dstdir.glob_files(f'{stem}*'))
+    errfiles = list(errdir.glob_files(f'{stem}*'))
+    # cache文件不算，不用管
+    check_files = [f for f in (dstfiles + errfiles) if not f.name.endswith('.cache.jsonl')]
+    if check_files:
+        if if_exists == 'skip':
+            return
+        elif if_exists == 'overwrite':
+            for f in check_files:
+                f.delete()
+        elif if_exists == 'error':
+            raise FileExistsError(f'目标文件已存在：{check_files}')
+        else:
+            return
+
+    # 3 处理数据
+    dstcgf = CacheJsonlGroupFiles(dstdir, infile.stem, tag, group_func, batch_size)
+    errcgf = CacheJsonlGroupFiles(errdir, infile.stem, tag + 'error', batch_size=batch_size)
+
+    for line in tqdm(infile.yield_line(), disable=total):
+        row = row_func(line)
+        if row:
+            dstcgf.append([row])
+        else:
+            errcgf.append([line])
+
+    dstcgf.save_all()
+    errcgf.save_all()
+    if total:
+        tprint(f'处理第{cur_idx}/{total}个文件：{infile.name} ==> {tag}')
+    else:
+        tprint(f'处理文件：{infile.name} ==> {tag}')

{pyxllib-0.3.100 → pyxllib-0.3.101}/pyxllib/prog/multiprocs.py

@@ -10,8 +10,7 @@ import os
 import subprocess
 import sys
 import time
-
-import fire
+import atexit
 
 from pyxllib.prog.pupil import tprint
 
@@ -19,19 +18,38 @@ from pyxllib.prog.pupil import tprint
 class MultiProcessLauncher:
     def __init__(self):
         self.workers = []
+        atexit.register(self.cleanup)  # 在程序退出时注册清理方法
 
-    def add_process(self, cmd, name=None, **kwargs):
-        # if isinstance(cmd, str):
-        #     cmd = cmd.split()  # todo 这种硬切分是不严谨的，但应急先这样处理
+    def _add_posix_process(self, cmd):
+        import ctypes
+        import signal
+
+        def _set_pdeathsig(sig=signal.SIGTERM):
+            """ Help function to ensure once parent process exits,
+            its children processes will automatically die on Linux.
+            """
+
+            def callable():
+                libc = ctypes.CDLL("libc.so.6")
+                return libc.prctl(1, sig)
 
-        if name is None:
-            name = cmd.split()[0]
+            return callable
 
-        p = subprocess.Popen(cmd)
-        # preexec_fn=_set_pdeathsig(signal.SIGTERM))
+        preexec_fn = _set_pdeathsig(signal.SIGTERM)
+        p = subprocess.Popen(cmd.split(), preexec_fn=preexec_fn)
+
+        return p
+
+    def add_process(self, cmd, name=None, **kwargs):
+        # 240329周五16:57，最新测试，好像不捕捉，主进程结束后子进程也是会自动结束的。跟我以前了解的机制似乎有点不一样。
+        #   但加着也算双重保险，不会出错吧。
+        if os.name == 'posix':
+            p = self._add_posix_process(cmd)
+        else:
+            p = subprocess.Popen(cmd)
 
         worker = {
-            'name': name,
+            'name': cmd.split()[0] if name is None else name,
             'process': p,
             'cmd': cmd,
             'status': 'running'  # 初始状态设为running
@@ -40,6 +58,15 @@ class MultiProcessLauncher:
 
         self.workers.append(worker)
 
+    def add_cur_py_process(self, cmd, name=None, **kwargs):
+        """ 添加当前python环境下的进程 """
+        if isinstance(cmd, str):
+            cmd = f'"{sys.executable}" {cmd}'
+        elif isinstance(cmd, list):
+            cmd = [sys.executable] + cmd
+
+        self.add_process(cmd, name, **kwargs)
+
     def manage(self):
         """ 进入交互管理界面 """
 
@@ -82,20 +109,13 @@ class MultiProcessLauncher:
             time.sleep(seconds)
             seconds = min(seconds + 1, 10)
 
-
-def trial_func(n):
-    for i in range(n):
-        print(i)
-        time.sleep(1)
-
-
-if __name__ == "__main__":
-
-    if len(sys.argv) > 1:
-        fire.Fire()
-    else:
-        launcher = MultiProcessLauncher()
-        launcher.add_process(f"{sys.executable} multiprocs.py trial_func 5", name="a")
-        launcher.add_process(f"{sys.executable} multiprocs.py trial_func 10", name="b")
-        launcher.add_process(f"{sys.executable} multiprocs.py trial_func 15", name="c")
-        launcher.wait_all()
+    def cleanup(self):
+        """ 在退出时终止所有子进程 """
+        for worker in self.workers:
+            process = worker['process']
+            if process.poll() is None:  # 如果进程还在运行
+                try:
+                    process.terminate()  # 尝试终止进程
+                    process.wait()  # 等待进程结束
+                except Exception as e:
+                    print(f"Error terminating process {worker['name']}: {e}")

{pyxllib-0.3.100 → pyxllib-0.3.101/pyxllib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyxllib
-Version: 0.3.100
+Version: 0.3.101
 Summary: 厦门理工模式识别团队通用python代码工具库
 Home-page: https://github.com/XLPRUtils/pyxllib
 Author: code4101

{pyxllib-0.3.100 → pyxllib-0.3.101}/setup.py

@@ -30,7 +30,7 @@ _dir = Path(__file__).parent
 
 setup(
     name='pyxllib',  # pip 安装时用的名字
-    version='0.3.100',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
+    version='0.3.101',  # 当前版本，每次更新上传到pypi都需要修改; 第4位版本号一般是修紧急bug
     author='code4101',
     author_email='877362867@qq.com',
     url='https://github.com/XLPRUtils/pyxllib',

{pyxllib-0.3.100 → pyxllib-0.3.101}/pyxllib.egg-info/requires.txt

@@ -21,14 +21,14 @@ Jinja2
 pandas>=0.23.4
 
 [xlai]
-visdom
-pynvml
-opsdroid-get-image-size
 xlcocotools
+opsdroid-get-image-size
+pynvml
 opencv-python
+visdom
 pillow
 
 [xlcv]
-opsdroid-get-image-size
 pillow
 opencv-python
+opsdroid-get-image-size