morebuiltins 0.0.1__py3-none-any.whl

morebuiltins/__init__.py

@@ -0,0 +1,7 @@
+__version__ = "0.0.1"
+__all__ = [
+    "morebuiltins.utils",
+    "morebuiltins.functools",
+    "morebuiltins.ipc",
+    "morebuiltins.request",
+]

morebuiltins/functools.py

@@ -0,0 +1,169 @@
+import time
+from concurrent.futures import ThreadPoolExecutor
+from functools import wraps
+from itertools import chain
+from typing import Optional, OrderedDict, Union
+from weakref import WeakSet
+
+__all__ = ["lru_cache_ttl", "threads"]
+
+
+def lru_cache_ttl(
+    maxsize: int,
+    ttl: Optional[Union[int, float]] = None,
+    controls=False,
+    auto_clear=True,
+    timer=time.time,
+):
+    """A Least Recently Used (LRU) cache with a Time To Live (TTL) feature.
+
+    Args:
+        maxsize (int): maxsize of cache
+        ttl (Optional[Union[int, float]], optional): time to live. Defaults to None.
+        controls (bool, optional): set cache/ttl_clean attributes. Defaults to False.
+        auto_clear (bool, optional): clear dead cache automatically. Defaults to True.
+        timer (callable, optional): Defaults to time.time.
+
+    Returns:
+        callable: decorator function
+
+    >>> import time
+    >>> # test ttl
+    >>> values = [1, 2]
+    >>> @lru_cache_ttl(1, 0.1)
+    ... def func1(i):
+    ...     return values.pop(0)
+    >>> [func1(1), func1(1), time.sleep(0.11), func1(1)]
+    [1, 1, None, 2]
+    >>> # test maxsize
+    >>> values = [1, 2, 3]
+    >>> func = lambda i: values.pop(0)
+    >>> func1 = lru_cache_ttl(2)(func)
+    >>> [func1(i) for i in [1, 1, 1, 2, 2, 2, 3, 3, 3]]
+    [1, 1, 1, 2, 2, 2, 3, 3, 3]
+    >>> # test auto_clear=True, with controls
+    >>> values = [1, 2, 3, 4]
+    >>> func = lambda i: values.pop(0)
+    >>> func1 = lru_cache_ttl(5, 0.1, controls=True, auto_clear=True)(func)
+    >>> [func1(1), func1(2), func1(3)]
+    [1, 2, 3]
+    >>> time.sleep(0.11)
+    >>> func1(3)
+    4
+    >>> len(func1.cache)
+    1
+    >>> # test auto_clear=False
+    >>> values = [1, 2, 3, 4]
+    >>> @lru_cache_ttl(5, 0.1, controls=True, auto_clear=False)
+    ... def func1(i):
+    ...     return values.pop(0)
+    >>> [func1(1), func1(2), func1(3)]
+    [1, 2, 3]
+    >>> time.sleep(0.11)
+    >>> func1(3)
+    4
+    >>> len(func1.cache)
+    3
+    """
+    cache: OrderedDict = OrderedDict()
+    move_to_end = cache.move_to_end
+    popitem = cache.popitem
+    next_clear_ts = timer()
+
+    def ttl_clean(expire):
+        for k, v in tuple(cache.items()):
+            if v[1] <= expire:
+                cache.pop(k, None)
+
+    def decorator(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            nonlocal next_clear_ts
+            now = timer()
+            if ttl is None:
+                expire_cleared = True
+            else:
+                expire_cleared = auto_clear and now > next_clear_ts
+                if expire_cleared:
+                    ttl_clean(now)
+                    next_clear_ts = now + ttl
+            # key = _make_key(args, kwargs, False)
+            key = hash(tuple(chain(args, kwargs.items())))
+            if key in cache:
+                if expire_cleared is True or now < cache[key][1]:
+                    # 1. key in cache; 2. cache is valid
+                    # set newest, return result
+                    move_to_end(key)
+                    return cache[key][0]
+            # call function
+            result = func(*args, **kwargs)
+            # ensure size
+            while len(cache) >= maxsize:
+                popitem(last=False)
+            if ttl is None:
+                # no need save expired ts
+                cache[key] = (result,)
+            else:
+                # setitem(key, (result, now + ttl))
+                cache[key] = (result, timer() + ttl)
+            # {key: (result, expired_ts)}
+            return result
+
+        if controls:
+            setattr(wrapped, "cache", cache)
+            setattr(wrapped, "ttl_clean", ttl_clean)
+        return wrapped
+
+    return decorator
+
+
+def threads(n: Optional[int] = None, executor_class=None, **kws):
+    """Quickly convert synchronous functions to be concurrency-able. (similar to madisonmay/Tomorrow)
+
+    >>> @threads(10)
+    ... def test(i):
+    ...     time.sleep(i)
+    ...     return i
+    >>> start = time.time()
+    >>> tasks = [test(i) for i in [0.1] * 5]
+    >>> len(test.pool._threads)
+    5
+    >>> len(test.tasks)
+    5
+    >>> for i in tasks:
+    ...     i.result() if hasattr(i, 'result') else i
+    0.1
+    0.1
+    0.1
+    0.1
+    0.1
+    >>> time.time() - start < 0.2
+    True
+    >>> len(test.pool._threads)
+    5
+    >>> len(test.tasks)
+    0
+    """
+    pool = (executor_class or ThreadPoolExecutor)(max_workers=n, **kws)
+    tasks: WeakSet = WeakSet()
+
+    def decorator(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            future = pool.submit(func, *args, **kwargs)
+            future.add_done_callback(lambda task: tasks.discard(task))
+            tasks.add(future)
+            return future
+
+        setattr(wrapped, "pool", pool)
+        setattr(wrapped, "tasks", tasks)
+        return wrapped
+
+    return decorator
+
+
+if __name__ == "__main__":
+    __name__ = "morebuiltins.functools"
+    import doctest
+
+    doctest.testmod()

morebuiltins/ipc.py

@@ -0,0 +1,376 @@
+import asyncio
+import pickle
+import sys
+from abc import ABC, abstractmethod
+from logging import LogRecord
+from logging.handlers import SocketHandler
+from typing import Any, Callable, Dict, Literal, Optional, Union
+
+__all__ = [
+    "IPCEncoder",
+    "JSONEncoder",
+    "PickleEncoder",
+    "SocketLogHandlerEncoder",
+    "SocketServer",
+    "SocketClient",
+]
+
+
+class IPCEncoder(ABC):
+    """An abstract base class for all encoders; implementing the necessary communication protocol requires only the definition of two abstract methods. Be mindful that varying header lengths will impact the maximum packaging size."""
+
+    # HEAD_LENGTH: Package length
+    # 1: 256 B, 2: 64 KB, 3: 16 MB, 4: 4 GB, 5: 1 TB, 6: 256 TB
+    HEAD_SIZE = 4
+    BYTEORDER: Literal["little", "big"] = "big"
+
+    @abstractmethod
+    def _dumps(self, raw: Any) -> bytes:
+        raise NotImplementedError
+
+    @abstractmethod
+    def _loads(self, raw: bytes) -> Any:
+        raise NotImplementedError
+
+    def dumps(self, raw: Any) -> bytes:
+        result = self._dumps(raw)
+        head = len(result).to_bytes(self.HEAD_SIZE, self.BYTEORDER)
+        return head + result
+
+    def loads(self, raw: Any) -> bytes:
+        return self._loads(raw)
+
+    def get_size(self, head: bytes):
+        return int.from_bytes(head, self.BYTEORDER)
+
+
+class JSONEncoder(IPCEncoder):
+    _DUMP_KWARGS: Dict = {}
+    _LOAD_KWARGS: Dict = {}
+    _ENCODING = "utf-8"
+
+    def __init__(self):
+        import json
+
+        self._encoder = json.dumps
+        self._decoder = json.loads
+        super().__init__()
+
+    def _dumps(self, raw: Any) -> bytes:
+        return self._encoder(raw, **self._DUMP_KWARGS).encode(self._ENCODING)
+
+    def _loads(self, raw: bytes) -> Any:
+        return self._decoder(raw.decode(self._ENCODING), **self._LOAD_KWARGS)
+
+
+class PickleEncoder(IPCEncoder):
+    _DUMP_KWARGS: Dict = {}
+    _LOAD_KWARGS: Dict = {}
+
+    def __init__(self):
+        super().__init__()
+
+    def _dumps(self, raw: Any) -> bytes:
+        return pickle.dumps(raw, **self._DUMP_KWARGS)
+
+    def _loads(self, raw: bytes) -> Any:
+        return pickle.loads(raw, **self._LOAD_KWARGS)
+
+
+class SocketLogHandlerEncoder(IPCEncoder):
+    """For a practical demonstration, refer to the test code: morebuiltins\ipc.py:_test_ipc_logging.
+
+    ```
+    async def _test_ipc_logging():
+        import logging
+
+        host = "127.0.0.1"
+        port = 8090
+        async with SocketServer(host=host, port=port, encoder=SocketLogHandlerEncoder()):
+            # socket logger demo
+            # ==================
+            logger = logging.getLogger("test_logger")
+            logger.setLevel(logging.DEBUG)
+            h = SocketHandler(host, port)
+            h.setLevel(logging.DEBUG)
+            logger.addHandler(h)
+            logger.info("test socket")
+            # ==================
+
+            # ensure test case
+            await asyncio.sleep(0.1)
+            assert pickle.loads(h.sock.recv(100000)[4:])["name"] == logger.name
+    ```
+    And provide a simple implementation for generating logs for coroutine code with Client usage.
+    """
+
+    _DUMP_KWARGS: Dict = {"protocol": 1}
+    _LOAD_KWARGS: Dict = {}
+
+    def __init__(self):
+        super().__init__()
+
+    def makePickle(self, d: Union[dict, LogRecord]):
+        if isinstance(d, LogRecord):
+            msg = d.getMessage()
+            d = dict(d.__dict__)
+            d["msg"] = msg
+            d["args"] = None
+            d["exc_info"] = None
+            d.pop("message", None)
+        return pickle.dumps(d, **self._DUMP_KWARGS)
+
+    def _dumps(self, raw: Union[LogRecord, dict]) -> bytes:
+        return self.makePickle(raw)
+
+    def _loads(self, raw: bytes) -> Any:
+        return pickle.loads(raw, **self._LOAD_KWARGS)
+
+
+class SocketServer:
+    """To see an example in action, view the test code: morebuiltins\ipc.py:_test_ipc.
+
+        ```
+    async def test_client(host="127.0.0.1", port=8090, encoder=None, cases=None):
+        async with SocketClient(host=host, port=port, encoder=encoder) as c:
+            for case in cases:
+                await c.send(case)
+                response = await c.recv()
+                if globals().get("print_log"):
+                    print("[Client]", "send:", repr(case), "=>", "recv:", repr(response))
+                assert case == response or str(case) == response, [case, response]
+            await c.send("[shutdown server]")
+
+
+    async def _test_ipc():
+        import platform
+
+        JSONEncoder._DUMP_KWARGS["default"] = str
+        for enc, cases in [
+            [PickleEncoder, [123, "123", None, {"a"}, ["a"], ("a",), {"a": 1}]],
+            [JSONEncoder, [123, "123", None, {"a"}, ["a"], {"a": 1}]],
+        ]:
+            encoder = enc()
+            if platform.system() == "Linux":
+                # test unix domain socket
+                print("Test Linux Unix Domain Socket")
+                host = "/tmp/uds.sock"
+                port = None
+                async with SocketServer(host=host, port=port, encoder=encoder):
+                    await test_client(host, port=None, encoder=encoder, cases=cases)
+
+            # test socket
+            host = "127.0.0.1"
+            port = 8090
+            async with SocketServer(host=host, port=port, encoder=encoder):
+                await test_client(host="127.0.0.1", port=8090, encoder=encoder, cases=cases)
+
+
+        ```
+    """
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 8090,
+        handler: Optional[Callable] = None,
+        encoder: Optional[IPCEncoder] = None,
+        connect_kwargs: Optional[dict] = None,
+    ):
+        if port is None and sys.platform == "win32":
+            raise SystemError("not support UDS(unix domain socket) on win32 platform")
+        self.host = host
+        self.port = port
+        self.handler = handler or self.default_handler
+        self.encoder: IPCEncoder = encoder or PickleEncoder()
+        self.connect_kwargs = connect_kwargs or {}
+
+        self.server: Optional[asyncio.base_events.Server] = None
+        self._shutdown_ev: Optional[asyncio.Event] = None
+
+    @staticmethod
+    async def default_handler(self: "SocketServer", item: Any):
+        if globals().get("print_log"):
+            print("[Server] recv:", repr(item), "=>", "send:", repr(item), flush=True)
+        if item == "[shutdown server]":
+            await self.close()
+        else:
+            return self.encoder.dumps(item)
+
+    async def connect_callback(
+        self,
+        reader: asyncio.StreamReader,
+        writer: asyncio.StreamWriter,
+    ) -> None:
+        head_size = self.encoder.HEAD_SIZE
+        need_await = asyncio.iscoroutinefunction(self.handler)
+        while self.is_serving() and not reader.at_eof():
+            head = await reader.read(head_size)
+            if len(head) < head_size:
+                raise RuntimeError()
+            content_length = self.encoder.get_size(head)
+            # read the whole package
+            head = await reader.read(content_length)
+            while len(head) < content_length:
+                head = head + await reader.read(content_length - len(head))
+            item = self.encoder._loads(head)
+            result = self.handler(self, item)
+            if need_await:
+                result = await result
+            if isinstance(result, bytes):
+                writer.write(result)
+                await writer.drain()
+        writer.close()
+        await writer.wait_closed()
+
+    async def close(self):
+        if self.is_serving():
+            self.server.close()
+            await self.server.wait_closed()
+            self._shutdown_ev.set()
+
+    async def start(self):
+        self._shutdown_ev = asyncio.Event()
+        if self.port:
+            self.server = await asyncio.start_server(
+                self.connect_callback,
+                host=self.host,
+                port=self.port,
+                **self.connect_kwargs,
+            )
+            await self.server.start_serving()
+        else:
+            self.server = await asyncio.start_unix_server(
+                self.connect_callback, path=self.host, **self.connect_kwargs
+            )
+
+    def is_serving(self):
+        return self.server and self.server.is_serving()
+
+    async def wait_closed(self):
+        if self._shutdown_ev:
+            await self._shutdown_ev.wait()
+
+    async def __aenter__(self):
+        await self.start()
+        return self
+
+    async def __aexit__(self, *_errors):
+        await self.close()
+
+
+class SocketClient:
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 8090,
+        encoder: Optional[IPCEncoder] = None,
+        connect_kwargs: Optional[dict] = None,
+    ):
+        self.host = host
+        self.port = port
+        self.encoder: IPCEncoder = encoder or PickleEncoder()
+        self.connect_kwargs = connect_kwargs or {}
+
+    async def __aenter__(self):
+        if self.port:
+            self.reader, self.writer = await asyncio.open_connection(
+                self.host, self.port, **self.connect_kwargs
+            )
+        else:
+            self.reader, self.writer = await asyncio.open_unix_connection(
+                path=self.host, **self.connect_kwargs
+            )
+        self.lock = asyncio.Lock()
+        return self
+
+    async def __aexit__(self, *_):
+        if self.writer:
+            self.writer.close()
+            await self.writer.wait_closed()
+
+    async def send(self, item: Any):
+        assert self.writer, "use `async with`"
+        async with self.lock:
+            self.writer.write(self.encoder.dumps(item))
+            await self.writer.drain()
+
+    async def recv(self) -> Any:
+        reader = self.reader
+        assert reader, "use `async with`"
+        async with self.lock:
+            assert not reader.at_eof()
+            head = await reader.read(self.encoder.HEAD_SIZE)
+            if len(head) < self.encoder.HEAD_SIZE:
+                raise RuntimeError()
+            content_length = self.encoder.get_size(head)
+            head = await reader.read(content_length)
+            while len(head) < content_length:
+                head = head + await reader.read(content_length - len(head))
+            return self.encoder.loads(head)
+
+
+async def test_client(host="127.0.0.1", port=8090, encoder=None, cases=None):
+    async with SocketClient(host=host, port=port, encoder=encoder) as c:
+        for case in cases:
+            await c.send(case)
+            response = await c.recv()
+            if globals().get("print_log"):
+                print("[Client]", "send:", repr(case), "=>", "recv:", repr(response))
+            assert case == response or str(case) == response, [case, response]
+        await c.send("[shutdown server]")
+
+
+async def _test_ipc():
+    import platform
+
+    JSONEncoder._DUMP_KWARGS["default"] = str
+    for enc, cases in [
+        [PickleEncoder, [123, "123", None, {"a"}, ["a"], ("a",), {"a": 1}]],
+        [JSONEncoder, [123, "123", None, {"a"}, ["a"], {"a": 1}]],
+    ]:
+        encoder = enc()
+        if platform.system() == "Linux":
+            # test unix domain socket
+            print("Test Linux Unix Domain Socket")
+            host = "/tmp/uds.sock"
+            port = None
+            async with SocketServer(host=host, port=port, encoder=encoder):
+                await test_client(host, port=None, encoder=encoder, cases=cases)
+
+        # test socket
+        host = "127.0.0.1"
+        port = 8090
+        async with SocketServer(host=host, port=port, encoder=encoder):
+            await test_client(host="127.0.0.1", port=8090, encoder=encoder, cases=cases)
+
+
+async def _test_ipc_logging():
+    import logging
+
+    host = "127.0.0.1"
+    port = 8090
+    async with SocketServer(host=host, port=port, encoder=SocketLogHandlerEncoder()):
+        # socket logger demo
+        # ==================
+        logger = logging.getLogger("test_logger")
+        logger.setLevel(logging.DEBUG)
+        h = SocketHandler(host, port)
+        h.setLevel(logging.DEBUG)
+        logger.addHandler(h)
+        logger.info("test socket")
+        # ==================
+
+        # ensure test case
+        await asyncio.sleep(0.1)
+        assert pickle.loads(h.sock.recv(100000)[4:])["name"] == logger.name
+
+
+def test():
+    globals().setdefault("print_log", True)  # local test show logs
+    for function in [_test_ipc, _test_ipc_logging]:
+        asyncio.get_event_loop().run_until_complete(function())
+
+
+if __name__ == "__main__":
+    test()