deepfos 1.1.60__py3-none-any.whl

deepfos/lib/nacos.py

@@ -0,0 +1,225 @@
+import asyncio
+import functools
+import json
+from collections import defaultdict
+from typing import TypedDict, Any, Tuple, Type, Dict
+
+from loguru import logger
+
+from deepfos import OPTION
+from deepfos.exceptions import APIResponseError
+from deepfos.lib.discovery import ServiceDiscovery
+from deepfos.lib.httpcli import AioHttpCli
+from deepfos.lib.utils import concat_url
+
+__all__ = ['Nacos']
+
+
+class InstanceHostInfo(TypedDict, total=False):
+    instanceId: str
+    ip: str
+    port: int
+    weight: float
+    healthy: bool
+    enabled: bool
+    ephemeral: bool
+    clusterName: str
+    serviceName: str
+    metadata: dict
+    instanceHeartBeatInterval: int
+    instanceIdGenerator: str
+    instanceHeartBeatTimeOut: int
+    ipDeleteTimeout: int
+
+
+class ResponseChecker:  # pragma: no cover
+    @classmethod
+    def expect(cls, response: Any) -> bool:
+        return True
+
+    @classmethod
+    def cast(cls, response: str, endpoint: str):
+        return response
+
+    @classmethod
+    def validate(cls, response: str, endpoint: str) -> Tuple[bool, Any]:
+        casted = cls.cast(response, endpoint)
+        return cls.expect(casted), casted
+
+
+class JsonResponse(ResponseChecker):  # pragma: no cover
+    @classmethod
+    def cast(cls, response: str, endpoint: str):
+        try:
+            return json.loads(response)
+        except (TypeError, ValueError):
+            logger.opt(lazy=True).exception(
+                f'Call api: {endpoint} failed. '
+                f'Response << {response} >> cannot be decoded as json.'
+            )
+            raise APIResponseError(
+                f'Call api: {endpoint} failed. '
+                f'Response << {response} >> cannot be decoded as json.'
+            )
+
+
+class Route:  # pragma: no cover
+    def __init__(self, method: str):
+        self.method = method
+
+    def __call__(
+        self,
+        endpoint,
+        response_checker: Type[ResponseChecker] = None,
+    ):
+        def execute(func):
+            request = getattr(AioHttpCli, self.method)
+            method = self.method.upper()
+            is_get = method == 'GET'
+
+            @functools.wraps(func)
+            async def call(*args, **kwargs):
+                ins = args[0]
+                url = concat_url(ins.base_url, endpoint)
+                body = func(ins, *args[1::], **kwargs)
+                logger.opt(lazy=True).debug(
+                    f"Sending request: {method} {url} "
+                    f"body: {repr(body)}"
+                )
+                req_args = {
+                    'headers': ins.header,
+                    'params' if is_get else 'data': body
+                }
+                response = await request(url, **req_args)
+                text = await response.text()
+                if response_checker is not None:
+                    ok, result = response_checker.validate(text, endpoint)
+                    if not ok:
+                        raise APIResponseError(
+                            f"Call API: {endpoint} failed. "
+                            f"Bad response because status is False. Detail: {text}."
+                        )
+                    else:
+                        return result
+                else:
+                    return text
+
+            return call
+
+        return execute
+
+
+get = Route(method='get')
+
+
+class NacosAPI:
+    def __init__(self):  # noqa
+        nacos_server: str = OPTION.nacos.server.lower()
+        if nacos_server.rstrip('/').endswith('nacos'):
+            base_url = nacos_server
+        else:
+            base_url = concat_url(nacos_server, 'nacos')
+
+        self.base_url = concat_url(base_url, 'v1')
+        self.header = {"Content-Type": "application/x-www-form-urlencoded"}
+
+    @get('ns/service/list', JsonResponse)
+    def list_service(self):  # pragma: no cover
+        return {
+            'pageNo': 1,
+            'pageSize': 1000000,
+        }
+
+    @get('ns/instance/list', JsonResponse)
+    def list_instance(
+        self,
+        service_name: str,
+        group: str = 'DEFAULT_GROUP',
+        namespace: str = 'public',
+        cluster: str = 'DEFAULT',
+    ):  # pragma: no cover
+        return {
+            'serviceName': service_name,
+            'groupName': group,
+            'namespaceId': namespace,
+            'clusters': cluster,
+        }
+
+
+class NacosCli(ServiceDiscovery):
+    def __init__(self):
+        super().__init__()
+        self._instance_lock: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+        self._server_lock = asyncio.Lock()
+        self._api = NacosAPI()
+        self._cares = set()
+        self._group = OPTION.nacos.group
+        self._ns = OPTION.nacos.namespace
+        self._cluster = OPTION.nacos.cluster
+
+    async def on_close(self):
+        self._cares.clear()
+        self.server_cache.clear()
+        self._instance_lock.clear()
+
+    async def on_startup(self):
+        pass
+
+    async def on_interval(self):
+        await self._update_cache()
+
+    async def _update_cache(self):
+        logger.opt(lazy=True).debug(f"Update cache for instance: {self._cares}")
+
+        await asyncio.gather(*(
+            self.update_instance_for_service(sn)
+            for sn in self._cares
+        ))
+
+    async def update_service_cache(self, server_name: str):
+        async with self._server_lock:
+            if server_name not in self.server_cache:
+                await self.update_services()
+
+    async def update_instance_cache(self, server_name):
+        self._cares.add(server_name)
+        await self.update_instance_for_service(server_name)
+
+    async def update_services(self):
+        new_services = frozenset((await self._api.list_service())['doms'])
+        cur_services = frozenset(self.server_cache.keys())
+
+        if added := new_services - cur_services:
+            for srv in added:
+                self.server_cache.__getitem__(srv)
+            logger.opt(lazy=True).debug(f"Added services: {added}")
+
+        if removed := cur_services - new_services:
+            for srv in removed:
+                self.server_cache.pop(srv)
+                self._instance_lock.pop(srv, None)  # noqa
+            logger.opt(lazy=True).debug(f"Removed services: {removed}")
+
+    async def update_instance_for_service(self, server_name: str):
+        async with self._instance_lock[server_name]:
+            instance_info = await self._api.list_instance(
+                server_name,
+                group=self._group,
+                namespace=self._ns,
+                cluster=self._cluster
+            )
+
+        host: InstanceHostInfo
+        cache = self.server_cache[server_name]
+
+        for host in instance_info['hosts']:
+            if not host['enabled']:
+                continue
+
+            addr = f"http://{host['ip']}:{host['port']}"
+            cache.add(addr)
+            if not host['healthy']:
+                cache.punish(addr)
+
+
+Nacos = NacosCli()

deepfos/lib/patch.py

@@ -0,0 +1,92 @@
+import functools
+from typing import Tuple, Union, Type, Callable
+from types import ModuleType
+import importlib
+from contextlib import contextmanager
+
+__all__ = [
+    'string_import',
+    'context_patch',
+    'patch'
+]
+
+
+def string_import(
+    importable: str,
+    check_obj: bool = False,
+) -> Tuple[ModuleType, str]:
+    if not importable.startswith('deepfos.'):
+        raise ValueError("Cannot patch object not belonged to deepfos package.")
+
+    module_str, obj_str = importable.rsplit('.', maxsplit=1)
+    module = importlib.import_module(module_str)
+
+    if check_obj and not hasattr(module, obj_str):
+        raise ValueError(f"Module: {module.__name__} has no attribute: {obj_str}")
+
+    return module, obj_str
+
+
+@contextmanager
+def context_patch(
+    importable: str,
+    patch_as: Union[Type, Callable, ModuleType]
+):
+    module, obj_str = string_import(importable)
+
+    if hasattr(module, obj_str):
+        obj_bak = getattr(module, obj_str)
+    else:
+        obj_bak = None
+
+    setattr(module, obj_str, patch_as)
+
+    try:
+        yield
+    finally:
+        if obj_bak is None:
+            delattr(module, obj_str)
+        else:
+            setattr(module, obj_str, obj_bak)
+
+
+def patch(
+    importable: str,
+    patch_as: Union[Type, Callable, ModuleType]
+):
+    # noinspection PyProtectedMember
+    """替换deepfos包的代码
+
+    装饰器，在被装饰的函数中，对应代码将被替换。
+    退出函数后，代码恢复。
+
+    Args:
+        importable: 需要替换的模块/函数/类
+        patch_as: 替换为对象
+
+    >>> from deepfos.lib.patch import patch
+    >>> from deepfos.lib.utils import _concat_url_single, concat_url
+    >>> def new_concat_url_single(a, b):
+    ...     return _concat_url_single(a, b) + '/patched'
+    >>> @patch('deepfos.lib.utils._concat_url_single', new_concat_url_single)
+    ... def pathced_func():
+    ...     print('in pathced_func')
+    ...     print(concat_url('a', 'b'))
+    >>> def main():
+    ...    print(concat_url('a', 'b'))
+    ...    pathced_func()
+    ...    print(concat_url('a', 'b'))
+    >>> main()
+    a/b
+    in pathced_func
+    a/b/patched
+    a/b
+    """
+    def inner(func):
+        @functools.wraps(func)
+        def wrap(*args, **kwargs):
+            with context_patch(importable, patch_as):
+                return func(*args, **kwargs)
+        return wrap
+
+    return inner

deepfos/lib/redis.py

@@ -0,0 +1,241 @@
+"""Redis相关"""
+
+from threading import Thread, Event
+from typing import Optional
+
+import redis
+from redis.exceptions import LockError, LockNotOwnedError
+from loguru import logger
+
+from deepfos.lib.decorator import cached_property
+from deepfos.options import OPTION
+from deepfos.exceptions import LockAcquireFailed
+
+__all__ = [
+    'RedisLock',
+    'RedisCli'
+]
+
+
+class _AbsRedisLock:  # pragma: no cover
+    # noinspection PyUnusedLocal
+    def __init__(
+        self,
+        key: str,
+        redis_client: redis.Redis = None,
+        renew_interval: int = 5,
+        expire_sec: int = 10,
+        raises: Exception = None,
+        blocking_timeout: Optional[int] = 0
+    ):
+        pass
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def acquire(self, timeout=0):
+        pass
+
+    def release(self):
+        pass
+
+
+def poll_event(
+    event: Event,
+    interval: int
+):
+    while not event.wait(interval):
+        yield
+
+
+class RedisLock:
+    """通过Redis实现的锁对象
+
+    Args:
+        redis_client: Redis对象
+        key: 锁名
+        renew_interval: 刷新有效时间间隔，默认为5秒
+        expire_sec: 有效时间，默认为10秒，需大于刷新间隔，否则刷新无效
+        raises: 获取锁失败时抛出的错误，仅使用with时有效
+        blocking_timeout: 获取锁的等待时间，默认为0，即不等待，为None时等待至获取到为止
+
+    Notes:
+        使用 ``with`` 获取锁时，如果获取失败，将直接报错。
+        使用 :meth:`aquire` 获取时，将返回 :class:`bool` 类型，表示是否获取成功。
+
+    .. admonition:: 示例
+
+        .. code-block:: python
+
+            with RedisLock('locked_key_a'):
+                do_something()
+
+            lock = RedisLock('locked_key_b')
+            if lock.aquire():
+                try:
+                    do_something()
+                finally:
+                    lock.release()
+
+    """
+    def __init__(
+        self,
+        key: str,
+        redis_client: redis.Redis = None,
+        renew_interval: int = 5,
+        expire_sec: int = 10,
+        raises: Exception = None,
+        blocking_timeout: Optional[int] = 0
+    ):
+        self._closed = None
+
+        if renew_interval > expire_sec:
+            raise ValueError("有效时间expire_sec需大于刷新间隔renew_interval，否则刷新无效。")
+
+        self.redis_client = redis_client or RedisCli().client
+        self.key = key
+        self.blocking_timeout = blocking_timeout
+        self.renew_interval = renew_interval
+        self.expire_sec = expire_sec
+        self.exc = raises
+
+    @property
+    def closed(self):
+        if self._closed is None:
+            return True
+        return self._closed.is_set()
+
+    @cached_property
+    def lock(self):
+        return self.redis_client.lock(
+            name=self.key,
+            timeout=self.expire_sec,
+            blocking_timeout=self.blocking_timeout,
+            thread_local=False
+        )
+
+    def __enter__(self):
+        if not self.closed:
+            logger.warning(f"RedisLock[key: {self.key}] already acquired.")
+            return
+
+        if self.lock.acquire(blocking=True):
+            self._closed = Event()
+            Thread(target=self.refresh_key, daemon=True).start()
+            return self
+
+        raise self.exc or LockAcquireFailed('Cannot aquire lock.')
+
+    do_hold = __enter__
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.release()
+
+    def acquire(self, timeout=0) -> bool:
+        """获取当前锁
+
+        Args:
+            timeout: 获取等待时间，默认为0，即不等待，为None时一直等待至得到锁
+        """
+        return self.lock.acquire(blocking=True, blocking_timeout=timeout)
+
+    def release(self):
+        """释放当前锁"""
+        self.stop_task()
+        try:
+            self.lock.release()
+        except (LockError, LockNotOwnedError):
+            logger.exception("")
+
+    def refresh_key(self):
+        self.lock.extend(self.expire_sec, replace_ttl=True)
+
+        for _ in poll_event(self._closed, self.renew_interval):
+            try:
+                self.lock.extend(self.expire_sec, replace_ttl=True)
+            except (LockError, LockNotOwnedError):
+                break
+
+    def owned(self):
+        """key是否被当前锁拥有"""
+        return self.lock.owned()
+
+    def locked(self):
+        """key是否被某个锁拥有"""
+        return self.lock.locked()
+
+    def stop_task(self):
+        if self.closed:
+            return
+
+        self._closed.set()
+
+    def __del__(self):
+        self.stop_task()
+
+
+if OPTION.general.dev_mode:
+    RedisLock = _AbsRedisLock
+
+
+class RedisCli:
+    """Redis对象
+
+    Args:
+        redis_url: redis地址，格式: redis://[[username]:[password]]@[host]:[port]/0，
+           若未提供，以OPTION.redis.url的值为默认值
+
+    """
+    _client: Optional[redis.Redis]
+
+    def __init__(self, redis_url: str = None):
+        self.redis_url = redis_url
+        self._client = None
+
+    @property
+    def client(self) -> redis.Redis:
+        if self._client is None:
+            self._client = redis.Redis.from_url(self.redis_url)
+        return self._client
+
+    def close(self):
+        if self._client is not None:
+            self._client.connection_pool.disconnect()
+            self._client = None
+
+    def __del__(self):
+        self.close()
+
+    def lock(
+        self,
+        key,
+        renew_interval: int = 5,
+        expire_sec: int = 10,
+        blocking_timeout: Optional[int] = 0
+    ) -> RedisLock:
+        """
+        提供设置了键名的redis维护锁
+
+        Args:
+            key:  键名
+            renew_interval: 刷新有效时间间隔，默认为5秒
+            expire_sec: 有效时间，默认为10秒，需大于刷新间隔，否则刷新无效
+            blocking_timeout: 获取锁的等待时间，默认为0，即不等待，为None时等待至获取到为止
+
+        Returns:
+            设置了键名key的RedisLock对象
+
+        .. admonition:: 示例
+
+            .. code-block:: python
+
+                rediscli = RedisCli()
+                with rediscli.lock('test_key'):
+                    ...
+
+        """
+        return RedisLock(key, self.client, renew_interval, expire_sec,
+                         blocking_timeout=blocking_timeout)