ldpy.base 0.0.1__py3-none-any.whl

ldpy/__init__.py

@@ -0,0 +1,8 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

ldpy/base/__init__.py

@@ -0,0 +1,7 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+

ldpy/base/ldenv.py

@@ -0,0 +1,62 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import os
+from typing import TypeVar, Union
+
+T = TypeVar('T')
+
+
+def get(key: 'str', _def: 'T' = None) -> 'Union[str, T]':
+    return os.getenv(key, _def)
+
+
+def get_as_str(key: 'str', _def: 'str' = '') -> 'str':
+    return get(key, _def)
+
+
+def get_as_int(key: 'str', _def: 'int' = 0) -> 'int':
+    v = get(key)
+    if v is None:
+        return _def
+    try:
+        return int(v)
+    except:
+        pass
+    try:
+        return int(float(v))
+    except:
+        pass
+    try:
+        return 1 if bool(v) else 0
+    except:
+        pass
+    return _def
+
+
+def get_as_bool(key: 'str', _def: 'bool' = False) -> 'bool':
+    v = get(key)
+    if not v:
+        return _def
+    try:
+        return True if int(v) else False
+    except:
+        pass
+    try:
+        return bool(v)
+    except:
+        pass
+    try:
+        return True if float(v) else False
+    except:
+        pass
+    v = v.lower()
+    # if v in ('on', 'enable', 'enabled'):
+    #     return True
+    if v in ('off', 'disable', 'disabled', 'null'):
+        return False
+    return True

ldpy/base/ldlog.py

@@ -0,0 +1,50 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import logging
+import time
+import traceback
+from typing import Optional
+
+
+class WithLog(object):
+    def __init__(self, name: str, logging=True, level=logging.INFO, ignore_exc: bool = False):
+        self._name = name
+        self._begin = 0
+        self._ignore_exc = ignore_exc
+        self._logging = logging
+        self._level = level
+
+    def _log_func(self, msg: 'str', stacklevel: 'int', exc_info: 'Optional[Exception]' = None):
+        if self._logging:
+            logging.log(self._level, msg, stacklevel=stacklevel + 1,
+                        exc_info=exc_info)
+        elif exc_info:
+            print(f'{msg}\n{"".join(traceback.format_exception(exc_info))}')
+
+        else:
+            print(msg)
+
+    def __enter__(self):
+        self._begin = time.time()
+        self._log_func(f' === do {self._name} begin', stacklevel=3)
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        cost = time.time() - self._begin
+        self._log_func(f' === do {self._name} end. cost:{cost:.6f}s', stacklevel=3,
+                       exc_info=exc_value)
+        if self._ignore_exc:
+            return True
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO,
+                        format='%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s')
+    with WithLog('abc', logging=False, ignore_exc=True):
+        1 / 0
+    with WithLog('abc', ignore_exc=True):
+        1 / 0

ldpy/base/ldpath.py

@@ -0,0 +1,29 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import logging
+import os
+import sys
+import tempfile
+
+
+class TempDir(object):
+    def __init__(self, dir='', prefix=''):
+        if not os.path.exists(dir):
+            os.makedirs(dir, exist_ok=True)
+
+        self._root_dir = dir
+        self._tdir = tempfile.TemporaryDirectory(dir=dir, prefix=prefix)
+        self.path = self._tdir.name
+        logging.info(f' === open temporary directory. dir:{self.path}')
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self._tdir.cleanup()
+        logging.info(f' === clean temporary directory. dir:{self.path}')

ldpy/base/ldsync.py

@@ -0,0 +1,147 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import threading
+from typing import Callable, Generic, Optional, TypeVar
+
+
+T = TypeVar('T')
+
+
+class _once(Generic[T]):
+    def __init__(self, func: 'Callable[[], T]'):
+        if not callable(func):
+            raise TypeError("the first argument must be callable")
+
+        self.__func = func
+
+        self.__done = False
+        self.__res = None
+
+    def reset(self):
+        self.__done = False
+
+    def done(self):
+        return self.__done
+
+    def __call__(self) -> T:
+        return self.call()
+
+    def call(self) -> T:
+        if self.__done:
+            return self.__res
+
+        self.__res = self.__func()
+        self.__done = True
+
+        return self.__res
+
+    def _get_res(self) -> 'Optional[T]':
+        return self.__res
+
+
+class Once(Generic[T]):
+    def __init__(self, func: 'Callable[[], T]'):
+        self.__once = _once(func)
+
+        self.__lock = threading.Lock()
+
+    def reset(self):
+        with self.__lock:
+            self.__once.reset()
+
+    def done(self):
+        return self.__once.done()
+
+    # @decorator.decorator
+    def __call__(self) -> T:
+        return self.call()
+
+    def call(self) -> T:
+        if self.done():
+            return self.__once._get_res()
+
+        with self.__lock:
+            return self.__once.call()
+
+
+class ThreadOnce(Generic[T]):
+    def __init__(self, func: 'Callable[[], T]'):
+        if not callable(func):
+            raise TypeError("the first argument must be callable")
+
+        self.__func = func
+        self.__data = threading.local()
+
+    def _get(self, force: bool = True) -> 'Optional[_once[T]]':
+        attr = 'value'
+        if hasattr(self.__data, attr):
+            return getattr(self.__data, attr)
+
+        if not force:
+            return None
+
+        v = _once(self.__func)
+        setattr(self.__data, attr, v)
+        return v
+
+    def reset(self):
+        v = self._get(force=False)
+        if not v:
+            return
+        v.reset()
+
+    def done(self) -> bool:
+        v = self._get(force=False)
+        return bool(v) and v.done()
+
+    def __call__(self) -> T:
+        return self.call()
+
+    def call(self) -> T:
+        v = self._get(force=True)
+        return v.call()
+
+
+def main():
+    __key = 'sequence'
+    __cache = {__key: 0}
+
+    def get_seq() -> int:
+        key = 'sequence'
+        __cache[key] += 1
+        return __cache[key]
+
+    def get_thread_run(fn: 'Callable[[], T]'):
+        def thread_run():
+            try:
+                import time
+                time.sleep(1)
+                print(fn(), fn(), fn())
+            except Exception as exc:
+                import traceback
+                print(traceback.format_exception(exc))
+        return thread_run
+
+    from concurrent.futures import ThreadPoolExecutor
+    num = 10
+
+    fn0 = Once(get_seq)
+    with ThreadPoolExecutor(max_workers=num) as executor:
+        for _ in range(num):
+            executor.submit(get_thread_run(fn0))
+    print(' === ')
+
+    fn1 = ThreadOnce(get_seq)
+    with ThreadPoolExecutor(max_workers=num) as executor:
+        for _ in range(num):
+            executor.submit(get_thread_run(fn1))
+    print(' === ')
+
+
+if __name__ == "__main__":
+    main()

ldpy/base/ldworker/__init__.py

@@ -0,0 +1,61 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) tanrenchong
+#
+
+
+import logging
+from .. import ldenv
+from .call_client import new_call
+
+DEFAULT_START_TIMEOUT = 600
+
+
+class CallWorkerBase(object):
+    def __init__(self):
+        pass
+
+    @classmethod
+    def name(cls):
+        n = cls.__name__
+        n = n.removesuffix('Worker')
+        n = n.lower()
+        n = n.removesuffix('_worker')
+        return n
+
+    @classmethod
+    def backlog(cls) -> int:
+        n = cls.name()
+        key0 = f'{n.upper()}_BACKLOG'
+        key1 = 'BACKLOG'
+        backlog = ldenv.get_as_int(key0) or ldenv.get_as_int(key1) or 0
+        logging.info(f'get call worker listen backlog succ. worker:{n}, '
+                     f'backlog:{backlog}')
+        return backlog
+
+    @classmethod
+    def worker_num(cls):
+        n = cls.name()
+        key = f'{n.upper()}_WORKER_NUM'
+        num = ldenv.get_as_int(key, 1)
+        logging.info(f'get call worker num succ. worker:{n}, num:{num}')
+        return num
+
+    @classmethod
+    def start_timeout(cls):
+        n = cls.name()
+        key0 = f'{n.upper()}_START_TIMEOUT'
+        key1 = 'START_TIMEOUT'
+        timeout = ldenv.get_as_int(key0) or ldenv.get_as_int(
+            key1) or DEFAULT_START_TIMEOUT
+        logging.info(f'get call worker start timeout succ. worker:{n}, '
+                     f'timeout:{timeout}')
+        return timeout
+
+
+__all__ = [
+    'DEFAULT_START_TIMEOUT',
+    'new_call',
+    'CallWorkerBase',
+]

ldpy/base/ldworker/call_client.py

@@ -0,0 +1,242 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+from collections.abc import Iterable, Mapping
+import logging
+import os
+import socket
+import select
+import time
+from typing import Type, TypeVar
+
+# import bytedance.context
+# from euler.errors import EulerError
+
+from . import conn, call_worker, call_master
+
+RES = TypeVar('RES')
+
+
+class CallClientImpl(call_worker.CallBase[RES]):
+    def __init__(self, worker_cls: 'Type[call_worker.CallWorker[RES]]'):
+        super().__init__(worker_cls)
+
+        pid = os.getpid()
+        self._logid = f'[client-{self._name}:{pid}]'
+
+        self._process_func = self._process
+
+    def start(self):
+        call_master.start(self)
+
+    def connect(self):
+        logid = self._logid
+        logging.info(f'{logid} connect to call worker begin')
+
+        self._process_func = self._build_process_func(self._process, self._client_midwares)
+
+        try:
+            now = time.time()
+            timeout = self._worker_cls.start_timeout()
+            ddl = now + timeout
+            while True:
+                try:
+                    c = self._connect(False)
+                    # logging.info(f'{logid} connect succ')
+
+                    req = call_worker.CallRequest(cmd=call_worker.CMD_INIT)
+                    req_raw = call_worker.CallRequest.encode(req)
+                    c.send(req_raw)
+                    # logging.info(f'{logid} send succ')
+
+                    rsp_raw = c.recv()
+                    rsp = call_worker.CallResponse[RES].decode(rsp_raw)
+                    logging.info(f'{logid} connect to call worker succ')
+                    return
+
+                except Exception as exc:
+                    if time.time() < ddl:
+                        continue
+
+                    logging.error(f'{logid} connect to call worker panic.',
+                                  exc_info=exc)
+                    raise exc
+        finally:
+            logging.info(f'{logid} connect to call worker end')
+
+    def _connect(self, timeout=1, need_log: 'bool' = True):
+        '''
+            panic if not log
+        '''
+
+        logid = self._logid
+
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.setblocking(False)
+        try:
+            sock.connect(self._worker_sock_path)
+        except BlockingIOError:
+            pass
+        except Exception as exc:
+            sock.close()
+
+            if not need_log:
+                logging.error(f'{logid} connect to call worker panic.',
+                              exc_info=exc)
+            raise exc
+
+        _, writeable, _ = select.select([], [sock], [], timeout)
+        if writeable:
+            # 再次检查连接状态
+            ret = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+            if ret == 0:
+                sock.setblocking(True)  # 恢复为阻塞模式以便后续操作
+                return conn.Conn(sock)
+
+            sock.close()
+            if need_log:
+                logging.error(
+                    f'{logid} connect to call worker fail. ret:{ret}')
+            raise ConnectionError(f'connect to call worker fail. ret:{ret}')
+
+        sock.close()
+        if need_log:
+            logging.error(f'{logid} connect to call worker timeout:{timeout}s')
+        raise TimeoutError(
+            f'connect to call worker timeout. timeout:{timeout}s')
+
+    def process(self, *args, **kwargs) -> 'RES':
+        return self._process_func(*args, **kwargs)
+
+    def _process(self, *args, **kwargs) -> 'RES':
+        # logid = bytedance.context.get('logid')
+        req = call_worker.CallRequest(args=list(args), kwargs=kwargs)
+        req_raw = call_worker.CallRequest.encode(req)
+
+        i = 0
+        max_retry = 3
+        while True:
+            i += 1
+            try:
+                rsp_raw = self._send_and_recv(req_raw)
+                break
+
+            except OSError as exc:
+                if exc.errno == 107 and i <= max_retry:
+                    continue
+
+                logging.error(f'{self._logid} send to call worker fail',
+                              exc_info=exc)
+                raise exc
+
+        rsp = call_worker.CallResponse[RES].decode(rsp_raw)
+        if rsp.exc:
+            raise rsp.exc
+
+        res = rsp.res
+        logging.info(f'call worker process succ. worker:{self._name}, '
+                     f'args:{_to_print(args)}, kwargs:{_to_print(kwargs)}, '
+                     f'res:{_to_print(res)}')
+        return res
+
+    def _send_and_recv(self, req0_raw: 'bytes') -> 'bytes':
+        c = self._connect()
+
+        try:
+            c.send(req0_raw)
+            return c.recv()
+
+        finally:
+            c.close()
+
+try:
+    from torch import Tensor
+except:
+    class Tensor(object):
+        dtype = 0
+        device = 0
+        shape = 0
+
+        def detach(self, *args, **kwargs): return self
+        def reshape(self, *args, **kwargs): return self
+        def numpy(self, *args, **kwargs): return self
+        def __getitem__(self, key): return self
+
+def _to_print(obj, seen=None):
+    """
+    将任意对象递归转换为 dict / list 结构，基础类型原样返回。
+
+    参数:
+        obj: 任意 Python 对象
+        seen: 用于检测循环引用的集合（内部使用，一般不需要手动提供）
+
+    返回:
+        转换后的 dict / list / 基础类型
+    """
+    if seen is None:
+        seen = set()
+
+    # 避免循环引用导致的无限递归
+    obj_id = id(obj)
+    if obj_id in seen:
+        # 循环引用时返回字符串标识（也可返回 None 或原对象，视需求而定）
+        return f"<recursion: {repr(obj)}>"
+    seen.add(obj_id)
+
+    try:
+        try:
+            if isinstance(obj, Tensor):
+                return f'<torch.Tensor(dtype={obj.dtype}, device={obj.device}, shape={obj.shape}, ' \
+                    f'sample={obj.detach()[:10].reshape(-1)[:10].numpy()})>'
+        except:
+            pass
+
+        # 1. 基础类型（不可变且无需转换）
+        if isinstance(obj, (int, float, str, bool, type(None))):
+            return obj
+
+        if isinstance(obj, bytes):
+            return f'<bytes(size={len(obj)}, sample={obj[:10]})>'
+
+        # 2. 字典或映射类型
+        if isinstance(obj, Mapping):
+            return {_to_print(k, seen): _to_print(v, seen) for k, v in obj.items()}
+
+        # 3. 可迭代对象（list, tuple, set, frozenset 等）
+        if isinstance(obj, Iterable) and not isinstance(obj, (str, bytes, dict)):
+            # 注意：str/bytes 已在上面的基础类型中处理，此处不会重复
+            return [_to_print(item, seen) for item in obj]
+
+        # 4. 自定义类实例（包括 namedtuple、dataclass、普通类）
+        # 优先使用 __dict__，若没有则尝试 __slots__
+        if hasattr(obj, '__dict__'):
+            # 直接使用实例字典
+            return _to_print(vars(obj), seen)
+
+        if hasattr(obj, '__slots__'):
+            # __slots__ 类没有 __dict__，手动构建字典
+            slots_dict = {
+                slot: getattr(obj, slot)
+                for slot in obj.__slots__ if hasattr(obj, slot)
+            }
+            return _to_print(slots_dict, seen)
+        # 5. 其他无法转换的对象（函数、模块、类型、None 等）
+        # 可根据需要改为 str(obj) 或抛出异常，这里保留原样
+        return obj
+
+    finally:
+        # 递归返回后移除 id，允许其他分支复用 seen
+        seen.remove(obj_id)
+
+
+def new_call(worker_cls: 'Type[call_worker.CallWorker[RES]]', start=False, connect=False) -> 'call_worker.CallClient[RES]':
+    cli = CallClientImpl[RES](worker_cls)
+    if start or connect:
+        cli.start()
+    if connect:
+        cli.connect()
+    return cli

ldpy/base/ldworker/call_master.py

@@ -0,0 +1,236 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import time
+import backgrounds
+import fcntl
+import logging
+import os
+import setproctitle
+import signal
+import socket
+import stat
+import sys
+from typing import Generic, List, TypeVar
+
+from .. import ldlog
+from . import call_service, call_worker
+
+RES = TypeVar('RES')
+
+
+class Lock(object):
+    def __init__(self, file_path: str) -> None:
+        self.lock_file_path = file_path
+        self.dir_path = os.path.dirname(file_path)
+        self.lock_fd = None
+
+    def try_lock(self):
+        lock_path = self.lock_file_path
+        lock_dir = os.path.dirname(lock_path)
+        try:
+            os.makedirs(lock_dir, 0o755, True)
+        except Exception as exc:
+            logging.error(f'os.makedirs fail. dir:{lock_dir}', exc_info=exc)
+
+        lock_fd = open(lock_path, 'w')
+        try:
+            fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            self.lock_fd = lock_fd
+            return True
+
+        except BlockingIOError:
+            lock_fd.close()
+            return False
+
+    def unlock(self):
+        lock_fd = self.lock_fd
+        self.lock_fd = None
+        if lock_fd:
+            fcntl.flock(lock_fd, fcntl.LOCK_UN)
+            lock_fd.close()
+
+
+class CallMaster(Generic[RES]):
+    def __init__(self, base: 'call_worker.CallBase[RES]'):
+        super().__init__()
+
+        self._base = base
+        worker_cls = base._worker_cls
+
+        pid = os.getpid()
+        self._logid = f'[master-{self._base._name}:{pid}]'
+
+        self._running = False
+        self._worker_num = max(worker_cls.worker_num(), 1)
+        self._children: 'List[int]' = []
+
+    def name(self): return self._base.name()
+
+    def _stop(self):
+        self._running = False
+
+    def run(self):
+        backgrounds.start()
+        self._running = True
+
+        def graceful_exit(sig, frame):
+            self._stop()
+
+        signal.signal(signal.SIGTERM, graceful_exit)
+        signal.signal(signal.SIGINT, graceful_exit)
+
+        lock = Lock(self._base._master_lock_path)
+        if not lock.try_lock():
+            logging.warning(f'{self._logid} exit because locked by another')
+            sys.exit(0)
+        logging.info(f'{self._logid} continue start after locked')
+
+        logging.info(f'{self._logid} run begin')
+        try:
+            self._run()
+        finally:
+            lock.unlock()
+            logging.info(f'{self._logid} run end')
+
+    def _run(self):
+        sock = self._listen()
+
+        try:
+            while self._running:
+                self._start_children(sock)
+                self._check_children_exit()
+                time.sleep(0.2)
+
+        finally:
+            sock.close()
+            self._wait_children()
+
+    def _wait_children(self):
+        logid = self._logid
+
+        try:
+            logging.info(f'{logid} kill all children at exit')
+            for pid in self._children:
+                os.kill(pid, signal.SIGTERM)
+
+            logging.info(f'{logid} wait all children at exit')
+
+            ddl = time.time() + 60
+            while len(self._children) > 0:
+                self._check_children_exit()
+                now = time.time()
+                if now > ddl:
+                    break
+                time.sleep(0.2)
+
+            for pid in self._children:
+                os.kill(pid, signal.SIGKILL)
+
+        except Exception as exc:
+            logging.error(f'{logid} wait all children panic at exit',
+                          exc_info=exc)
+        finally:
+            logging.info(f'{logid} wait all children end at exit')
+
+    def _check_children_exit(self):
+        logid = self._logid
+
+        for pid in self._children:
+            try:
+                ret_pid, status = os.waitpid(pid, os.WNOHANG)
+                if ret_pid != pid:  # 子进程已结束
+                    continue
+                code = os.WEXITSTATUS(status)
+                logging.info(f'{logid} wait child exit. pid:{pid}, code:{code}')
+                self._children.remove(pid)
+            except ChildProcessError:
+                # 指定的子进程不存在
+                code = 'unknow'
+                logging.info(f'{logid} wait child exit. pid:{pid}, code:{code}')
+                self._children.remove(pid)
+
+    def _start_children(self, sock: socket.socket):
+        num = self._worker_num
+        while len(self._children) < num:
+            pid = call_service.start(num, sock, self._base)
+            self._children.append(pid)
+            logging.info(f'{self._logid} start call worker succ. pid:{pid}')
+
+    def _listen(self):
+        sock_path = self._base._worker_sock_path
+        if os.path.exists(sock_path):
+            os.remove(sock_path)
+
+        backlog = self._base._worker_cls.backlog()
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.bind(sock_path)
+        sock.listen(backlog)
+
+        logging.info(f'{self._logid} call worker listen succ. sock:{sock_path}')
+        return sock
+
+
+def _is_socket(fd: int):
+    try:
+        # 1. 检查是否是 socket
+        mode = os.fstat(fd).st_mode
+        if stat.S_IFMT(mode) != stat.S_IFSOCK:
+            return False
+
+        # 2. 创建临时 socket 对象（不拥有 fd 的所有权）
+        s = socket.socket(fileno=fd)
+        # 3. 获取 SO_ACCEPTCONN 选项
+        opt = s.getsockopt(socket.SOL_SOCKET, socket.SO_ACCEPTCONN)
+        # 4. 将 fd 从 socket 对象中分离，防止对象销毁时关闭 fd
+        s.detach()
+        return opt == 1
+    except OSError:
+        # fd 无效或不存在，忽略
+        return False
+
+    return True
+
+
+def close_all_socket():
+    try:
+        max_fd = os.sysconf('SC_OPEN_MAX')
+    except (ValueError, OSError):
+        max_fd = 1024   # 常见系统的安全上限
+
+    # skip stdin(0), stdout(1), stderr(2)
+    for fd in range(3, max_fd):
+        try:
+            if _is_socket(fd):
+                os.close(fd)
+        except:
+            pass
+
+
+def start(base: 'call_worker.CallBase[RES]'):
+    # wait + fork 2次，避免僵尸进程
+    pid = os.fork()
+    if pid > 0:
+        # in parent
+        os.waitpid(pid, 0)
+        return pid
+
+    pid = os.fork()
+    if pid > 0:
+        # in parent
+        sys.exit(0)
+
+    # in child
+    proc_title = f'call: master [{base.name()}]'
+    setproctitle.setproctitle(proc_title)
+
+    with ldlog.WithLog('close_all_socket'):
+        close_all_socket()
+
+    mgr = CallMaster(base)
+    mgr.run()
+    sys.exit(0)

ldpy/base/ldworker/call_service.py

@@ -0,0 +1,165 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import threading
+import time
+import backgrounds
+import logging
+import os
+import sys
+import signal
+import socket
+from typing import Generic, TypeVar
+import psutil
+import setproctitle
+
+from .. import ldlog
+from . import conn, call_worker
+
+
+RES = TypeVar('RES')
+
+
+class Service(Generic[RES]):
+    def __init__(self, worker_num: 'int', sock: 'socket.socket', base: 'call_worker.CallBase[RES]'):
+        super().__init__()
+
+        self._base = base
+        worker_cls = base._worker_cls
+
+        ppid = os.getppid()
+        pid = os.getpid()
+
+        self._logid = f'[worker-{worker_cls.name()}:{pid}]'
+
+        with ldlog.WithLog(f'init_worker_object {worker_cls.name()}'):
+            w = worker_cls()
+
+        self._ppid = ppid
+        self._name = worker_cls.name()
+        self._worker_cls = worker_cls
+        self._worker = w
+        self._worker_num = worker_num
+
+        self._process_func = base._build_process_func(w.process, base._server_midwares)
+
+        self._sock = sock
+        self._running = False
+
+    def _stop(self):
+        self._running = False
+
+    def run(self):
+        backgrounds.start()
+        self._running = True
+
+        def graceful_exit(sig, frame):
+            self._stop()
+
+        signal.signal(signal.SIGTERM, graceful_exit)
+        signal.signal(signal.SIGINT, graceful_exit)
+
+        th = threading.Thread(target=self._thread_check_parent)
+        th.daemon = True
+        th.start()
+
+        for func in self._base._post_fork_funcs:
+            func()
+
+        self._sock.settimeout(1.0)
+
+        logid = self._logid
+        logging.info(f'{logid} call worker run begin')
+        try:
+            while self._running:
+                try:
+                    sock, addr = self._sock.accept()
+                except socket.timeout:
+                    # 超时后继续循环，检查 should_exit
+                    continue
+
+                # logging.info(f'{logid} accept conn succ. addr:{addr}')
+                c = conn.Conn(sock)
+
+                try:
+                    self._handle_conn(c)
+                except Exception as exc:
+                    logging.info(f'{logid} call worker handle conn panic', exc_info=exc)
+                finally:
+                    c.close()
+        finally:
+            logging.info(f'{logid} call worker run end')
+
+    def _thread_check_parent(self):
+        ppid = self._ppid
+        logid = self._logid
+
+        logging.info(f'{logid} thread check parent begin. ppid:{ppid}')
+
+        pps = psutil.Process(ppid)
+        while self._running:
+            try:
+                status = pps.status()
+                if status in (psutil.STATUS_DEAD, psutil.STATUS_STOPPED):
+                    logging.warning(f'{logid} parent is not running now. '
+                                    f'ppid:{ppid}')
+                    self._stop()
+            except psutil.NoSuchProcess:
+                logging.warning(f'{logid} parent is not running now. '
+                                f'ppid:{ppid}')
+                self._stop()
+            finally:
+                time.sleep(1)
+        logging.info(f'{logid} thread check parent end. ppid:{ppid}')
+
+    def _handle_conn(self, c: 'conn.Conn'):
+        req_raw = c.recv()
+        req = call_worker.CallRequest.decode(req_raw)
+
+        if req.command == call_worker.CMD_INIT:
+            rsp = call_worker.CallResponse[RES]()
+        elif req.command == call_worker.CMD_CALL:
+            rsp = self._process_request(req)
+        else:
+            msg = f'invalid command. cmd:{req.command}, worker:{self._name}'
+            logging.error(msg)
+            exc = Exception(msg)
+            rsp = call_worker.CallResponse[RES](exc=exc)
+
+        rsp_raw = call_worker.CallResponse.encode(rsp)
+        c.send(rsp_raw)
+
+    def _process_request(self, req: 'call_worker.CallRequest') -> 'call_worker.CallResponse[RES]':
+        logid = self._logid
+
+        with ldlog.WithLog(f'call worker process [{self._name}]'):
+            try:
+                # res = self._worker.process(*req.args, **req.kwargs)
+                res = self._process_func(*req.args, **req.kwargs)
+                logging.info(f'{logid} process request succ')
+                return call_worker.CallResponse[RES](res=res)
+
+            except Exception as exc:
+                logging.error(f'{logid} process request panic', exc_info=exc)
+                return call_worker.CallResponse[RES](exc=exc)
+
+
+def start(worker_num: 'int', sock: 'socket.socket', base: 'call_worker.CallBase[RES]') -> 'int':
+    pid = os.fork()
+    if pid > 0:
+        # in parent
+        return pid
+
+    # in child
+    s = Service(worker_num, sock, base)
+
+    worker_cls = base._worker_cls
+    proc_title = f'call: worker [{worker_cls.name()}]'
+    setproctitle.setproctitle(proc_title)
+
+    s.run()
+    sys.exit(0)

ldpy/base/ldworker/call_test.py

@@ -0,0 +1,58 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+
+
+import logging
+import os
+import sys
+from typing import Any, Callable, Dict, Optional, Tuple
+
+from .. import ldlog
+
+from . import CallWorkerBase, new_call
+
+
+def log(source: 'str', globals: 'Optional[Dict[str, Any]]' = None, locals: 'Optional[Dict[str, Any]]' = None):
+    logging.info(f'{source} = {eval(source, globals, locals)}')
+
+
+class TestWorker(CallWorkerBase):
+    def process(self, a: 'int', b: 'int') -> 'Tuple[float, int]':
+        return a / b, a % b
+
+
+def server_midware(*args, next: 'Callable', **kwargs):
+    with ldlog.WithLog('server_midware'):
+        return next(*args, **kwargs)
+
+def client_midware(*args, next: 'Callable', **kwargs):
+    with ldlog.WithLog('client_midware'):
+        return next(*args, **kwargs)
+
+
+def main():
+    cli = new_call(TestWorker)
+
+    cli.add_client_midware(client_midware)
+    cli.add_server_midware(server_midware)
+    cli.add_post_fork_func(lambda: logging.info(f'worker has started. pid:{os.getpid()}'))
+
+    cli.start()
+    cli.connect()
+
+    locals = {'cli': cli}
+    log('cli.process(1, 2)', locals=locals)
+    log('cli.process(a=1, b=0)', locals=locals)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO,
+                        format='%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s')
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.stderr.write("\033[1;31moperation cancelled by user\033[0m\n")
+        sys.exit(-1)

ldpy/base/ldworker/call_worker.py

@@ -0,0 +1,126 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) distroy
+#
+# pyright: reportInvalidTypeVarUse=false
+
+
+import multiprocessing.reduction
+import os
+from typing import Callable, Dict, Generic, List, Optional, Protocol, Type, TypeVar
+
+PICKLER = multiprocessing.reduction.ForkingPickler
+
+RES = TypeVar('RES')
+
+
+class CallWorker(Protocol[RES]):
+    @classmethod
+    def name(cls) -> str: ...
+    @classmethod
+    def backlog(cls) -> int: ...
+    @classmethod
+    def worker_num(cls) -> int: ...
+    @classmethod
+    def start_timeout(cls) -> int: ...
+
+    def process(self, *args, **kwargs) -> 'RES': ...
+
+
+class CallMidware(Protocol[RES]):
+    def __call__(self, *args, next: 'Callable[..., RES]', **kwargs) -> 'RES':
+        ...
+
+
+class CallClient(Protocol[RES]):
+    def name(self) -> str: ...
+    def add_post_fork_func(self, func: 'Callable[[], None]'): ...
+    def add_server_midware(self, mw: 'CallMidware[RES]'): ...
+    def add_client_midware(self, mw: 'CallMidware[RES]'): ...
+    def start(self): ...
+    def connect(self): ...
+    def process(self, *args, **kwargs) -> 'RES': ...
+
+
+class CallBase(Generic[RES]):
+    def __init__(self, worker_cls: 'Type[CallWorker[RES]]'):
+        super().__init__()
+
+        name = worker_cls.name()
+        name = name.removesuffix('_worker')
+        self._worker_cls = worker_cls
+        self._name = name
+
+        cwd = os.path.abspath(os.getcwd())
+        cache_dir = os.path.join(cwd, f'.cache/ld-call-worker')
+
+        self._cache_dir = cache_dir
+        self._master_lock_path = os.path.join(cache_dir, f'{name}-master.lock')
+        self._worker_sock_path = os.path.join(cache_dir, f'{name}-worker.sock')
+
+        self._post_fork_funcs: 'List[Callable[[], None]]' = []
+        self._server_midwares: 'List[CallMidware[RES]]' = []
+        self._client_midwares: 'List[CallMidware[RES]]' = []
+
+    def name(self) -> str:
+        return self._name
+
+    def add_post_fork_func(self, func: 'Callable[[], None]'):
+        self._post_fork_funcs.append(func)
+
+    def add_server_midware(self, mw: 'CallMidware[RES]'):
+        self._server_midwares.append(mw)
+
+    def add_client_midware(self, mw: 'CallMidware[RES]'):
+        self._client_midwares.append(mw)
+
+    def _build_process_func(self, func: 'Callable[..., RES]', mws: 'List[CallMidware[RES]]'):
+        i = len(mws)
+        def get_next_func(next, mw):
+            return lambda *args, **kwargs: mw(*args, next=next, **kwargs)
+        while i > 0:
+            i -= 1
+            mw = mws[i]
+            func = get_next_func(func, mw)
+        return func
+
+
+
+CMD_INIT = 0
+CMD_CALL = 1
+
+
+class CallRequest(object):
+    def __init__(self, cmd=CMD_CALL, args: 'List' = [], kwargs: 'Dict' = {}) -> None:
+        super().__init__()
+
+        self.command = cmd
+        self.args = args
+        self.kwargs = kwargs
+
+    @classmethod
+    def decode(cls, raw: 'bytes') -> 'CallRequest':
+        return PICKLER.loads(raw)
+
+    @classmethod
+    def encode(cls, obj: 'CallRequest') -> 'bytes':
+        raw = PICKLER.dumps(obj)
+        return bytes(raw)
+
+
+class CallResponse(Generic[RES]):
+    def __init__(self, exc: 'Optional[Exception]' = None, res: 'Optional[RES]' = None) -> None:
+        super().__init__()
+
+        self.exc = exc
+        self.res = res
+
+    @classmethod
+    def decode(cls, raw: 'bytes') -> 'CallResponse[RES]':
+        return PICKLER.loads(raw)
+
+    @classmethod
+    def encode(cls, obj: 'CallResponse[RES]') -> 'bytes':
+        raw = PICKLER.dumps(obj)
+        return bytes(raw)

ldpy/base/ldworker/conn.py

@@ -0,0 +1,49 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) tanrenchong
+#
+
+
+import socket
+
+
+PADDING = b'x'[0]
+
+class Conn(object):
+    def __init__(self, sock: 'socket.socket') -> None:
+        self.sock = sock
+
+    def recv_exact(self, n: 'int'):
+        sock = self.sock
+        data = b''
+        while len(data) < n:
+            chunk = sock.recv(n - len(data))
+            if not chunk:
+                raise ConnectionError("connection has been closed")
+            data += chunk
+        return data
+
+    def recv(self):
+        """full message: 1 byte padding + 4 bytes size + raw"""
+        raw0 = self.recv_exact(1)
+        pad = raw0[0]
+        if pad != PADDING:
+            raise Exception(f'recv invalid padding. want:{PADDING}, got:{pad}')
+        raw1 = self.recv_exact(4)
+        size = int.from_bytes(raw1, 'big', signed=True)
+
+        raw = self.recv_exact(size)
+        return raw
+
+    def send(self, raw: bytes):
+        """full message: 1 byte padding + 4 bytes size + raw"""
+        raw0 = bytes([PADDING])
+        size = len(raw)
+        raw1 = size.to_bytes(4, 'big', signed=True)
+
+        buff = b''.join([raw0, raw1, raw])
+        self.sock.sendall(buff)
+
+    def close(self):
+        self.sock.close()

ldpy_base-0.0.1.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: ldpy.base
+Version: 0.0.1
+Summary: No description
+Author-email: distroy <distroy@live.com>
+Requires-Dist: requests>=2.25.0
+Requires-Dist: setproctitle>=1.3.7
+Requires-Dist: pywin32>=300; sys_platform == "win32"

ldpy_base-0.0.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+ldpy/__init__.py,sha256=Y-njSLV1W3Mbdh21wNcFytFIGkQzD98XBEYsSChBkXs,143
+ldpy/base/__init__.py,sha256=dVaLmNrWWMZTf84uo3ZkD0dVXevQhZK-aMHX8mpHddM,78
+ldpy/base/ldenv.py,sha256=MVc8llqrePByFy2jEopCQaC122weVj7INrLUinA8LUk,1125
+ldpy/base/ldlog.py,sha256=y-pT2AHH8QvZ2s-Xi-xSwAQoybKZzmz-z1ZtGhrrITc,1503
+ldpy/base/ldpath.py,sha256=NAuW0ZZh5rrEGLOOZZcyZvJ419-1yFrzCdiTQBrtffA,698
+ldpy/base/ldsync.py,sha256=T30t4pFB7V9VQNmXa58PKaahCMFM-LOMeXwRXy2ID40,3234
+ldpy/base/ldworker/__init__.py,sha256=k3R3uQTUMRjPVraHFLC-zgBb0xYCwrhz027K9ARvSCQ,1479
+ldpy/base/ldworker/call_client.py,sha256=C0INbRiOCCzcFTbKve_Ar1_hfAZYByH0ey-rU-Y-YHg,7722
+ldpy/base/ldworker/call_master.py,sha256=hit9t5L28dm1RNHHElwXdmqe2Dotes22iIvzZOVYKOE,6443
+ldpy/base/ldworker/call_service.py,sha256=JWAlqqxDIIAPBvgn3kFg42AaruZrw1-sX5xDBcap5Po,4856
+ldpy/base/ldworker/call_test.py,sha256=XxsyGGK9lM7YLZEOyt6OVDz8c5UJQOCdwxo86FLVhjE,1522
+ldpy/base/ldworker/call_worker.py,sha256=iA_gHeZUr2v3WRcdF0jNadu7-unV6QlNfbNaj8zefGg,3506
+ldpy/base/ldworker/conn.py,sha256=RhwzlQy7_RHLovNk4Y1C7koJWqBgNuJkf4HJvDUOJfM,1235
+ldpy_base-0.0.1.dist-info/METADATA,sha256=yBk2Fs_J5iqb8brTdGxDxTSKP6kASOs02ig6ovYBB0E,238
+ldpy_base-0.0.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+ldpy_base-0.0.1.dist-info/top_level.txt,sha256=dkG_F4jYkh6CS6gdpQeV_gKW3yaJBvl1ukUu0zo2KA8,5
+ldpy_base-0.0.1.dist-info/RECORD,,

ldpy_base-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ldpy_base-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+ldpy