aspyx-service 0.11.2__py3-none-any.whl

aspyx_service/registries.py

@@ -0,0 +1,241 @@
+"""
+registries for components in aspyx service
+"""
+import re
+import threading
+from abc import abstractmethod
+import time
+from typing import Optional
+
+import consul
+
+from aspyx.di.configuration import inject_value
+from aspyx.util import StringBuilder
+from aspyx.di import on_init
+from .healthcheck import HealthCheckManager, HealthStatus
+
+from .server import Server
+from .service import ComponentRegistry, Channel, ChannelInstances, ServiceManager, ComponentDescriptor, Component, ChannelAddress
+
+class ConsulComponentRegistry(ComponentRegistry):
+    """
+    A specialized registry using consul.
+    A polling mechanism is used to identify changes in the component health.
+    """
+    # constructor
+
+    def __init__(self, port: int, consul: consul.Consul):
+        self.port = port
+        self.ip = Server.get_local_ip()
+        self.running = False
+        self.consul = consul
+        self.watchdog = None
+        self.interval = 5
+        self.last_index = {}
+        self.component_addresses : dict[str, dict[str, ChannelInstances]] = {} # comp -> channel -> address
+        self.watch_channels : list[Channel] = []
+        self.watchdog_interval = 5
+        self.healthcheck_interval = "10s"
+        self.healthcheck_timeout= "5s"
+        self.healthcheck_deregister = "5m"
+
+    # injections
+
+    @inject_value("consul.watchdog.interval", default=5)
+    def set_watchdog_interval(self, interval):
+        self.watchdog_interval = interval
+
+    @inject_value("consul.healthcheck.interval", default="10s")
+    def set_healthcheck_interval(self, interval):
+        self.healthcheck_interval = interval
+
+    @inject_value("consul.healthcheck.timeout", default="3s")
+    def set_healthcheck_timeout(self, interval):
+        self.healthcheck_timeout = interval
+
+    @inject_value("consul.healthcheck.deregister", default="5m")
+    def set_healthcheck_deregister(self, interval):
+        self.healthcheck_deregister = interval
+
+    # lifecycle hooks
+
+    @on_init()
+    def setup(self):
+        # create consul client
+
+        self.running = True
+
+        # start thread
+
+        self.watchdog = threading.Thread(target=self.watch_consul, daemon=True)
+        self.watchdog.start()
+
+    def inform_channels(self, old_address: ChannelInstances, new_address: Optional[ChannelInstances]):
+        for channel in self.watch_channels:
+            if channel.address is old_address:
+                channel.set_address(new_address)
+
+
+    def watch_consul(self):
+        while self.running:
+            # check services
+
+            for component, old_addresses in self.component_addresses.items():
+                old_addresses = dict(old_addresses) # we will modify it...
+                new_addresses = self.fetch_addresses(component, wait="1s")
+
+                # compare
+
+                changed = False
+                for channel_name, address in new_addresses.items():
+                    service_address = old_addresses.get(channel_name, None)
+                    if service_address is None:
+                        ServiceManager.logger.info("new %s address for %s", channel_name, component)
+                        changed = True
+                    else:
+                        if service_address != address:
+                            changed = True
+
+                            ServiceManager.logger.info("%s address for %s changed", channel_name, component)
+
+                            # inform channels
+
+                            self.inform_channels(service_address, address)
+
+                        # delete
+
+                        del old_addresses[channel_name]
+
+                # watchout for deleted addresses
+
+                for channel_name, address in old_addresses.items():
+                    ServiceManager.logger.info("deleted %s address for %s", channel_name, component)
+
+                    changed = True
+
+                    # inform channel
+
+                    self.inform_channels(address, None)
+
+                # replace ( does that work while iterating )
+
+                if changed:
+                    self.component_addresses[component] = new_addresses
+
+            # time to sleep
+
+            time.sleep(self.watchdog_interval)
+
+    @abstractmethod
+    def watch(self, channel: Channel) -> None:
+        self.watch_channels.append(channel)
+
+   # public
+
+    def register_service(self, name, service_id, health: str, tags=None, meta=None) -> None:
+        self.consul.agent.service.register(
+            name=name,
+            service_id=service_id,
+            address=self.ip,
+            port=self.port,
+            tags=tags or [],
+            meta=meta or {},
+            check=consul.Check().http(
+                url=f"http://{self.ip}:{self.port}{health}",
+                interval=self.healthcheck_interval,
+                timeout=self.healthcheck_timeout,
+                deregister=self.healthcheck_deregister)
+            )
+
+    def deregister(self, descriptor: ComponentDescriptor[Component]) -> None:
+        name = descriptor.name
+
+        service_id = f"{self.ip}:{self.port}:{name}"
+
+        self.consul.agent.service.deregister(service_id)
+
+    def stop(self):
+        self.running = False
+        if self.watchdog is not None:
+            self.watchdog.join()
+            self.watchdog = None
+
+    # private
+
+    def fetch_addresses(self, component: str, wait=None) -> dict[str, ChannelInstances]:
+        addresses : dict[str, ChannelInstances] = {} # channel name -> ServiceAddress
+
+        index, nodes = self.consul.health.service(component, passing=True, index=self.last_index.get(component, None), wait=wait)
+        self.last_index[component] = index
+
+        for node in nodes:
+            service = node["Service"]
+
+            meta = service.get('Meta')
+
+            channels = meta.get("channels").split(",")
+
+            for channel in channels:
+                match = re.search(r"([\w-]+)\((.*)\)", channel)
+
+                channel_name = match.group(1)
+                url = match.group(2)
+
+                address = addresses.get(channel, None)
+                if address is None:
+                    address = ChannelInstances(component=component, channel=channel_name)
+                    addresses[channel] = address
+
+                address.urls.append(url)
+
+        # done
+
+        return addresses
+
+    # implement ComponentRegistry
+
+    def register(self, descriptor: ComponentDescriptor[Component], addresses: list[ChannelAddress]):
+        name = descriptor.name
+
+        id = f"{self.ip}:{self.port}:{name}"
+
+        builder = StringBuilder()
+        first = True
+        for address in addresses:
+            if not first:
+                builder.append(",")
+
+            builder.append(address.channel).append("(").append(address.uri).append(")")
+
+            first = False
+
+        addresses = str(builder)
+
+        self.register_service(name, id, descriptor.health, tags =["component"], meta={"channels": addresses})
+
+    def get_addresses(self, descriptor: ComponentDescriptor) -> list[ChannelInstances]:
+        component_addresses = self.component_addresses.get(descriptor.name, None)
+        if component_addresses is None:
+            component_addresses = self.fetch_addresses(descriptor.name)
+
+            # only cache if non-empty
+
+            if component_addresses:
+                self.component_addresses[descriptor.name] = component_addresses
+
+        return list(component_addresses.values())
+
+    # 200–299	passing	Service is healthy (OK, Created, No Content…)
+    # 429	warning	Rate limited or degraded
+    # 300–399	warning	Redirects interpreted as potential issues
+    # 400–499	critical	Client errors (Bad Request, Unauthorized…)
+    # 500–599	critical	Server errors (Internal Error, Timeout…)
+    # Other / No response	critical	Timeout, connection refused, etc.
+
+    def map_health(self, health: HealthCheckManager.Health) -> int:
+        if health.status is HealthStatus.OK:
+            return 200
+        elif health.status is HealthStatus.WARNING:
+            return 429
+        else:
+            return 500

aspyx_service/restchannel.py

@@ -0,0 +1,313 @@
+"""
+rest channel implementation
+"""
+import inspect
+import re
+from dataclasses import is_dataclass
+
+from typing import get_type_hints, TypeVar, Annotated, Callable, get_origin, get_args, Type
+from pydantic import BaseModel
+
+from aspyx.reflection import DynamicProxy, Decorators
+from aspyx.util import get_serializer
+
+from .channels import HTTPXChannel
+from .service import channel, ServiceCommunicationException
+
+T = TypeVar("T")
+
+class BodyMarker:
+    pass
+
+Body = lambda t: Annotated[t, BodyMarker]
+
+class QueryParamMarker:
+    pass
+
+QueryParam = lambda t: Annotated[t, QueryParamMarker]
+
+# decorators
+
+def rest(url=""):
+    """
+    mark service interfaces to add a url prefix
+
+    Args:
+        url: prefix that will be added to all urls
+    """
+    def decorator(cls):
+        Decorators.add(cls, rest, url)
+
+        return cls
+    return decorator
+
+def get(url: str):
+    """
+    methods marked with `get` will be executed by calling a http get request.
+
+    Args:
+        url: the url
+    """
+    def decorator(cls):
+        Decorators.add(cls, get, url)
+
+        return cls
+    return decorator
+
+
+def post(url: str):
+    """
+    methods marked with `post` will be executed by calling a http get request.
+    The body parameter should be marked with `Body(<param>)`
+
+    Args:
+        url: the url
+    """
+    def decorator(cls):
+        Decorators.add(cls, post, url)
+
+        return cls
+
+    return decorator
+
+def put(url: str):
+    """
+    methods marked with `put` will be executed by calling a http put request.
+
+    Args:
+        url: the url
+    """
+    def decorator(cls):
+        Decorators.add(cls, put, url)
+
+        return cls
+
+    return decorator
+
+def delete(url: str):
+    """
+    methods marked with `delete` will be executed by calling a http delete request.
+
+    Args:
+        url: the url
+    """
+    def decorator(cls):
+        Decorators.add(cls, delete, url)
+
+        return cls
+
+    return decorator
+
+def patch(url: str):
+    """
+    methods marked with `patch` will be executed by calling a http patch request.
+
+    Args:
+        url: the url
+    """
+    def decorator(cls):
+        Decorators.add(cls, patch, url)
+
+        return cls
+
+    return decorator
+
+@channel("rest")
+class RestChannel(HTTPXChannel):
+    """
+    A rest channel executes http requests as specified by the corresponding decorators and annotations,
+    """
+    __slots__ = [
+        "signature",
+        "url_template",
+        "type",
+        "calls",
+        "return_type",
+        "path_param_names",
+        "query_param_names",
+        "body_param_name"
+    ]
+
+    # local class
+
+    class Call:
+        # slots
+
+        __slots__ = [
+            "type",
+            "url_template",
+            "path_param_names",
+            "body_param_name",
+            "query_param_names",
+            "return_type",
+            "signature",
+            "body_serializer"
+        ]
+
+        # constructor
+
+        def __init__(self, type: Type, method : Callable):
+            self.signature = inspect.signature(method)
+
+            type_hints = get_type_hints(method)
+
+            param_names = list(self.signature.parameters.keys())
+            param_names.remove("self")
+
+            prefix = ""
+            if Decorators.has_decorator(type, rest):
+                prefix = Decorators.get_decorator(type, rest).args[0]
+
+            # find decorator
+
+            self.type = "get"
+            self.url_template = ""
+
+            decorators = Decorators.get_all(method)
+
+            for decorator in [get, post, put, delete, patch]:
+                descriptor = next((descriptor for descriptor in decorators if descriptor.decorator is decorator), None)
+                if descriptor is not None:
+                    self.type = decorator.__name__
+                    self.url_template = prefix + descriptor.args[0]
+
+            # parameters
+
+            self.path_param_names = set(re.findall(r"{(.*?)}", self.url_template))
+
+            for param_name in self.path_param_names:
+                param_names.remove(param_name)
+
+            hints = get_type_hints(method, include_extras=True)
+
+            self.body_param_name = None
+            self.query_param_names = set()
+
+            for param_name, hint in hints.items():
+                if get_origin(hint) is Annotated:
+                    metadata = get_args(hint)[1:]
+
+                    if BodyMarker in metadata:
+                        self.body_param_name = param_name
+                        self.body_serializer = get_serializer(type_hints[param_name])
+                        param_names.remove(param_name)
+                    elif QueryParamMarker in metadata:
+                        self.query_param_names.add(param_name)
+                        param_names.remove(param_name)
+
+            # check if something is missing
+
+            if param_names:
+                # check body params
+                if self.type in ("post", "put", "patch"):
+                    if self.body_param_name is None:
+                        candidates = [
+                            (name, hint)
+                            for name, hint in hints.items()
+                            if name not in self.path_param_names
+                        ]
+                        # find first dataclass or pydantic argument
+                        for name, hint in candidates:
+                            typ = hint
+                            if get_origin(typ) is Annotated:
+                                typ = get_args(typ)[0]
+                            if (
+                                    (isinstance(typ, type) and issubclass(typ, BaseModel))
+                                    or is_dataclass(typ)
+                            ):
+                                self.body_param_name = name
+                                self.body_serializer = get_serializer(type_hints[name])
+                                param_names.remove(name)
+                                break
+
+            # the rest are query params
+
+            for param in param_names:
+                self.query_param_names.add(param)
+
+            # return type
+
+            self.return_type = type_hints['return']
+
+    # constructor
+
+    def __init__(self):
+        super().__init__()
+
+        self.calls : dict[Callable, RestChannel.Call] = {}
+
+    # internal
+
+    def get_call(self, type: Type ,method: Callable):
+        call = self.calls.get(method, None)
+        if call is None:
+            call = RestChannel.Call(type, method)
+            self.calls[method] = call
+
+        return call
+
+    # override
+
+    async def invoke_async(self, invocation: 'DynamicProxy.Invocation'):
+        call = self.get_call(invocation.type, invocation.method)
+
+        bound = call.signature.bind(self, *invocation.args, **invocation.kwargs)
+        bound.apply_defaults()
+        arguments = bound.arguments
+
+        # url
+
+        url = call.url_template.format(**arguments)
+        query_params = {k: arguments[k] for k in call.query_param_names if k in arguments}
+        body = {}
+        if call.body_param_name is not None:
+            body = call.body_serializer(arguments.get(call.body_param_name))#self.to_dict(arguments.get(call.body_param_name))
+
+        # call
+
+        try:
+            result = None
+            if call.type in ["get", "put", "delete"]:
+                result = await self.request_async(call.type, self.get_url() + url, params=query_params, timeout=self.timeout)
+
+            elif call.type == "post":
+                result = await self.request_async("post", self.get_url() + url, params=query_params, json=body, timeout=self.timeout)
+
+            return self.get_deserializer(invocation.type, invocation.method)(result.json())
+        except ServiceCommunicationException:
+            raise
+
+        except Exception as e:
+            raise ServiceCommunicationException(f"communication exception {e}") from e
+
+    def invoke(self, invocation: DynamicProxy.Invocation):
+        call = self.get_call(invocation.type, invocation.method)
+
+        bound = call.signature.bind(self,*invocation.args, **invocation.kwargs)
+        bound.apply_defaults()
+        arguments = bound.arguments
+
+        # url
+
+        url = call.url_template.format(**arguments)
+        query_params = {k: arguments[k] for k in call.query_param_names if k in arguments}
+        body = {}
+        if call.body_param_name is not None:
+            body = call.body_serializer(arguments.get(call.body_param_name))#self.to_dict(arguments.get(call.body_param_name))
+
+        # call
+
+        try:
+            result = None
+            if call.type in ["get", "put", "delete"]:
+                result = self.request(call.type, self.get_url() + url, params=query_params, timeout=self.timeout)
+
+            elif call.type == "post":
+                result = self.request( "post", self.get_url() + url, params=query_params, json=body, timeout=self.timeout)
+
+            return self.get_deserializer(invocation.type, invocation.method)(result.json())
+        except ServiceCommunicationException:
+            raise
+
+        except Exception as e:
+            raise ServiceCommunicationException(f"communication exception {e}") from e