splight-lib 3.3.7__tar.gz → 4.0.0.dev0__tar.gz

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: splight-lib
-Version: 3.3.7
+Version: 4.0.0.dev0
 Summary: Library for public use. Splight
 Home-page: UNKNOWN
 Author: Splight

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/setup.py

@@ -14,7 +14,7 @@ test_requires = [
 
 setup(
     name="splight-lib",
-    version="3.3.7",
+    version="4.0.0.dev0",
     author="Splight",
     author_email="factory@splight-ae.com",
     packages=find_packages(),

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/splight_lib/component/abstract.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import traceback
 from functools import partial
 from tempfile import NamedTemporaryFile
 from time import sleep
@@ -79,6 +80,8 @@ class HealthCheckProcessor:
         self._running = True
         while self._running:
             if not self._engine.healthcheck():
+                exc = self._engine.get_last_exception()
+                self._log_exception(exc)
                 self._logger.error(
                     "Healthcheck task failed.", tags=LogTags.RUNTIME
                 )
@@ -94,6 +97,11 @@ class HealthCheckProcessor:
     def stop(self):
         self._running = False
 
+    def _log_exception(self, exc: Optional[Exception]):
+        if exc:
+            message = "".join(traceback.format_tb(exc.__traceback__))
+            self._logger.exception(message)
+
 
 class SplightBaseComponent:
     def __init__(

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/splight_lib/component/exceptions.py

@@ -20,7 +20,10 @@ class MissingBindingCallback(Exception):
 
 class MissingRoutineCallback(Exception):
     def __init__(self, routine_name: str, action: str):
-        msg = f"Missing method associated with a Routine: {routine_name} {action}_handler event"
+        msg = (
+            f"Missing method associated with a Routine: {routine_name} "
+            f"{action}_handler event"
+        )
         super().__init__(msg)
 
 
@@ -40,7 +43,3 @@ class MissingCommandCallback(Exception):
     def __init__(self, method_name: str):
         msg = f"Missing method {method_name} associated with a Command"
         super().__init__(msg)
-
-
-class LogsStreamerError(Exception):
-    pass

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/splight_lib/execution.py

@@ -6,7 +6,7 @@ from functools import wraps
 from subprocess import Popen as DefaultPopen
 from threading import Event, Lock
 from threading import Thread as DefaultThread
-from typing import Any, Callable, List, Tuple, Union
+from typing import Any, Callable, List, Optional, Tuple, Union
 
 from splight_lib.abstract.client import AbstractClient
 from splight_lib.logging._internal import LogTags, get_splight_logger
@@ -240,8 +240,20 @@ class Thread(DefaultThread):
     def __init__(self, target: Callable, args: Tuple = (), **kwargs) -> None:
         target = self.store_result(target)
         self.result = Empty()
+        self._exc = None
         super().__init__(target=target, args=args, name=target, **kwargs)
 
+    @property
+    def exc(self):
+        return self._exc
+
+    def run(self):
+        try:
+            super().run()
+        except Exception as exc:
+            self._exc = exc
+            raise exc
+
     def store_result(self, func: Callable) -> Callable:
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -342,9 +354,20 @@ class ExecutionClient(AbstractClient):
         return self._scheduler.unschedule(job)
 
     def healthcheck(self):
-        return all(
+        is_alive = all(
             [
                 p.is_alive() or p.exit_ok()
                 for p in self.processes + self.threads
             ]
         )
+        return is_alive
+
+    def get_last_exception(self) -> Optional[Exception]:
+        """Get the last exception thrown in one of the threads.
+        It assumes that there is only one thread that crashed
+        Also, only works for the thread not the processes.
+        """
+        broken_thread = [x.exc for x in self.threads if x.exc]
+        if broken_thread:
+            return broken_thread[0]
+        return None

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/splight_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: splight-lib
-Version: 3.3.7
+Version: 4.0.0.dev0
 Summary: Library for public use. Splight
 Home-page: UNKNOWN
 Author: Splight

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/splight_lib.egg-info/SOURCES.txt

@@ -41,10 +41,6 @@ splight_lib/client/datalake/abstract.py
 splight_lib/client/datalake/builder.py
 splight_lib/client/datalake/local_client.py
 splight_lib/client/datalake/remote_client.py
-splight_lib/client/grpc/__init__.py
-splight_lib/client/grpc/client.py
-splight_lib/client/grpc/decorators.py
-splight_lib/client/grpc/reflector.py
 splight_lib/client/hub/__init__.py
 splight_lib/client/hub/abstract.py
 splight_lib/client/hub/client.py
@@ -53,7 +49,6 @@ splight_lib/communication/event_handler.py
 splight_lib/component/__init__.py
 splight_lib/component/abstract.py
 splight_lib/component/exceptions.py
-splight_lib/component/log_streamer.py
 splight_lib/component/spec.py
 splight_lib/logging/__init__.py
 splight_lib/logging/_internal.py

{splight-lib-3.3.7 → splight-lib-4.0.0.dev0}/splight_lib.egg-info/requires.txt

@@ -6,10 +6,6 @@ cryptography==35.0.0
 email-validator==1.3.1
 furl==2.1.3
 geojson-pydantic==0.5.0
-grpc-interceptor==0.15.0
-grpcio-reflection==1.54.2
-grpcio-tools==1.54.2
-grpcio==1.54.2
 hexbytes==0.2.2
 httpx==0.23.3
 mergedeep==1.3.4

splight-lib-3.3.7/splight_lib/client/grpc/client.py

@@ -1,71 +0,0 @@
-from typing import Callable, Optional, Tuple
-
-import grpc
-
-from splight_lib.client.grpc.decorators import retry_streaming
-from splight_lib.client.grpc.reflector import GrpcReflectionClient
-
-
-class LogsGRPCError(Exception):
-    pass
-
-
-class MissingGRPCService(Exception):
-    pass
-
-
-class SplightGRPCClient:
-    AUTHORIZATION: str = "authorization"
-    _SERVICE_NAME: str = None
-
-    def __init__(self, grpc_host: str, secure_channel: bool = True):
-        if not self._SERVICE_NAME:
-            raise MissingGRPCService("Missing parameter service_name")
-
-        if secure_channel:
-            self._channel = grpc.secure_channel(
-                grpc_host, grpc.ssl_channel_credentials()
-            )
-        else:
-            self._channel = grpc.insecure_channel(grpc_host)
-
-        self._reflector = GrpcReflectionClient()
-        self._reflector.load_protocols(
-            self._channel, symbols=[self._SERVICE_NAME]
-        )
-        self._stub = self._reflector.service_stub_class(self._SERVICE_NAME)(
-            self._channel
-        )
-        self._auth_header: Optional[Tuple[str, str]] = None
-
-    def set_authorization_header(self, access_id: str, secret_key: str):
-        self._auth_header = (
-            SplightGRPCClient.AUTHORIZATION,
-            f"Splight {access_id} {secret_key}",
-        )
-
-
-class LogsGRPCClient(SplightGRPCClient):
-    _SERVICE_NAME: str = "LogsService"
-    _LOG_ENTRY: str = "LogEntry"
-
-    def __init__(self, grpc_host: str, secure_channel: bool = True):
-        super().__init__(grpc_host, secure_channel=secure_channel)
-        self._log_entry = self._reflector.message_class(self._LOG_ENTRY)
-
-    @retry_streaming(times=5)
-    def stream_logs(self, log_generator: Callable, component_id: str):
-        try:
-            self._stub.stash_log_entry(
-                self._parse_log_message(log_generator(), component_id),
-                metadata=[self._auth_header],
-            )
-        except grpc.RpcError as exc:
-            raise LogsGRPCError("Unable to stream component logs") from exc
-
-    def _parse_log_message(self, message_iterator: str, component_id: str):
-        for message in message_iterator:
-            yield self._log_entry(
-                message=message,
-                component_id=component_id,
-            )

splight-lib-3.3.7/splight_lib/client/grpc/decorators.py

@@ -1,34 +0,0 @@
-from time import sleep
-from typing import Callable
-
-import grpc
-
-from splight_lib.logging._internal import get_splight_logger
-
-logger = get_splight_logger()
-
-
-class GRPCRetryError(Exception):
-    pass
-
-
-def retry_streaming(
-    times: int = 3, delay: float = 0.5, delay_factor: float = 2
-):
-    def decorator(func: Callable):
-        def wrapper(*args, **kwargs):
-            try:
-                func(*args, **kwargs)
-            except grpc.RpcError as e:
-                logger.error(f"Error: {e}")
-                if times == 0:
-                    raise GRPCRetryError("Max retries exceeded")
-                sleep(delay)
-                logger.warn(f"Retrying {func.__name__}, {times} retries left")
-                retry_streaming(times - 1, delay * delay_factor)(func)(
-                    *args, **kwargs
-                )
-
-        return wrapper
-
-    return decorator

splight-lib-3.3.7/splight_lib/client/grpc/reflector.py

@@ -1,320 +0,0 @@
-"""A gRPC reflection client.
-
-This module provides a programmatic interface for getting grpc class types
-discovered via gRPC reflection service.
-
-See:
-    https://grpc.io/ for more information on gRPC in general.
-
-    https://github.com/grpc/grpc/blob/master/doc/server-reflection.md for more
-    information on gRPC service reflection.
-
-Code taken from
-    https://github.com/sparky8512/yagrc
-"""
-from collections import deque
-
-from google.protobuf import descriptor_pb2, descriptor_pool, message_factory
-from google.protobuf.internal import enum_type_wrapper
-from grpc_reflection.v1alpha import reflection_pb2, reflection_pb2_grpc
-
-# This is somewhat arbitrary, and is just here to prevent hang in the case of
-# dead network connection that the client side still believes to be open.
-QUERY_TIMEOUT = 10
-"""Max wait time for reflection query completion, in seconds."""
-
-
-class ServiceError(Exception):
-    """Error reported by reflection service."""
-
-
-def __stub_init__(self, channel):
-    """The __init__ method for service stub classes."""
-    for (
-        name,
-        call,
-        full_name,
-        out_serializer,
-        in_serializer,
-    ) in self._methods:  # pylint: disable=protected-access
-        setattr(
-            self,
-            name,
-            getattr(channel, call)(
-                full_name,
-                request_serializer=out_serializer,
-                response_deserializer=in_serializer,
-            ),
-        )
-
-
-def _list_services(stub):
-    responses = stub.ServerReflectionInfo(
-        iter([reflection_pb2.ServerReflectionRequest(list_services="")]),
-        timeout=QUERY_TIMEOUT,
-    )
-    for response in responses:
-        if response.HasField("error_response"):
-            raise ServiceError(response.error_response.error_message)
-        for service in response.list_services_response.service:
-            yield service.name
-
-
-def list_services(channel):
-    """Get list of fully qualified service names via reflection.
-
-    Args:
-        channel (grpc.Channel): The RPC channel to use.
-
-    Raises:
-        ServiceError: Reflection service responded with an error.
-        grpc.RpcError: Lower level RPC error.
-    """
-    stub = reflection_pb2_grpc.ServerReflectionStub(channel)
-    return list(_list_services(stub))
-
-
-def enum_from_descr(proto):
-    """Get enum class from enum descriptor.
-
-    Args:
-        proto (google.protobuf.descriptor.EnumDescriptor): Enum descriptor.
-    """
-    return enum_type_wrapper.EnumTypeWrapper(proto)
-
-
-class GrpcReflectionEngine:
-    """Implementation class for gRPC reflection client.
-
-    This class is not meant to be used directly. Instead, use
-    `GrpcReflectionClient` or `yagrc.importer.GrpcImporter`
-    """
-
-    def __init__(self):
-        self.methods_by_file = {}
-        self.pool = descriptor_pool.DescriptorPool()
-        self._factory = message_factory.MessageFactory(self.pool)
-
-    def load_protocols(self, channel, filenames=None, symbols=None):
-        """Implementation of `GrpcReflectionClient.load_protocols`"""
-        stub = reflection_pb2_grpc.ServerReflectionStub(channel)
-
-        requests = []
-        if filenames:
-            requests.extend(
-                reflection_pb2.ServerReflectionRequest(file_by_filename=name)
-                for name in filenames
-            )
-        if symbols:
-            requests.extend(
-                reflection_pb2.ServerReflectionRequest(
-                    file_containing_symbol=symbol
-                )
-                for symbol in symbols
-            )
-        if not requests:
-            requests.extend(
-                reflection_pb2.ServerReflectionRequest(
-                    file_containing_symbol=name
-                )
-                for name in _list_services(stub)
-                if name != "grpc.reflection.v1alpha.ServerReflection"
-            )
-
-        protos = {}
-        traversed = set()
-        while requests:
-            responses = stub.ServerReflectionInfo(
-                iter(requests), timeout=QUERY_TIMEOUT
-            )
-            deps = set()
-            for response in responses:
-                if response.HasField("error_response"):
-                    raise ServiceError(response.error_response.error_message)
-                for (
-                    desc_bytes
-                ) in response.file_descriptor_response.file_descriptor_proto:
-                    proto = descriptor_pb2.FileDescriptorProto.FromString(  # pylint: disable=no-member
-                        desc_bytes
-                    )
-                    traversed.add(proto.name)
-                    deps.update(proto.dependency)
-                    protos[proto.name] = proto
-                    self.methods_by_file[proto.name] = {
-                        service.name: service.method
-                        for service in proto.service
-                    }
-            deps -= traversed
-            requests = [
-                reflection_pb2.ServerReflectionRequest(file_by_filename=dep)
-                for dep in deps
-            ]
-            # prevent unsatisfied deps from looping forever
-            traversed.update(deps)
-
-        names = deque(protos.keys())
-        traversed = set()
-        while names:
-            name = names[0]
-            traversed.add(name)
-            # raises KeyError if unsatisfied dep:
-            proto = protos[name]
-            deps = set(proto.dependency) - traversed
-            if deps:
-                names = deque(x for x in names if x not in deps)
-                names.extendleft(deps)
-            else:
-                del names[0]
-                self.pool.Add(proto)
-
-        return protos.keys()
-
-    def file_descriptor(self, name):
-        """Get file descriptor for a proto file that has been loaded.
-
-        Args:
-            name (str): The file name of the .proto file, including path.
-
-        Returns:
-             google.protobuf.descriptor.FileDescriptor: The descriptor for the
-                specified file.
-
-        Raises:
-            KeyError: File has not been loaded.
-        """
-        return self.pool.FindFileByName(name)
-
-    def message_from_descr(self, proto):
-        """Get message class from message descriptor.
-
-        Args:
-            proto (google.protobuf.descriptor.Descriptor): Message descriptor.
-        """
-        message = self._factory.GetPrototype(proto)
-        return message
-
-    def gen_stub_class(self, service, method_protos):
-        """Get service stub class from service descriptor.
-
-        Args:
-            service (google.protobuf.descriptor.ServiceDescriptor): Service
-                descriptor.
-            method_protos (iterable): The
-                `google.protobuf.descriptor_pb2.MethodDescriptorProto` objects
-                for the methods in the service.
-        """
-        stub_methods = []
-        dep_descrs = []
-        for method_proto in method_protos:
-            method = service.methods_by_name[method_proto.name]
-            channel_call = "_".join(
-                "stream" if x else "unary"
-                for x in [
-                    method_proto.client_streaming,
-                    method_proto.server_streaming,
-                ]
-            )
-            dep_descrs.extend([method.input_type, method.output_type])
-            stub_methods.append(
-                (
-                    method.name,
-                    channel_call,
-                    "/{}/{}".format(service.full_name, method.name),
-                    self.message_from_descr(
-                        method.input_type
-                    ).SerializeToString,
-                    self.message_from_descr(method.output_type).FromString,
-                )
-            )
-        class_name = service.name + "Stub"
-        return (
-            type(
-                class_name,
-                (),
-                {"_methods": stub_methods, "__init__": __stub_init__},
-            ),
-            dep_descrs,
-        )
-
-
-class GrpcReflectionClient:
-    """Client to discover protocol types via gRPC service reflection.
-
-    Protocol files must be loaded via `load_protocols` prior to getting any of
-    the class types.
-
-    Note that for nested types (fields, oneofs, etc) other than nested message
-    types, the top-level message type should be requested and the nested types
-    can be accessed on it via attribute.
-    """
-
-    def __init__(self):
-        self._engine = GrpcReflectionEngine()
-
-    def load_protocols(self, channel, filenames=None, symbols=None):
-        """Load a set of proto files to use later for protocol types.
-
-        Load one or more specified files and/or files containing specified
-        symbols, as well as the transitive dependency files of those files.
-
-        If neither `filenames` nor `symbols` is specified, then files
-        containing the symbols for all services advertised by the RPC server,
-        other than the reflection service, are loaded.
-
-        This method may be called multiple times to load additional proto
-        files; however, it is not recommended to use channels that point to
-        different servers unless it is known that the symbols either do not
-        overlap or contain the same exact version across all servers.
-
-        The caller-supplied channel is only used within the context of this
-        method call, so the caller is free to close it afterwards.
-
-        Args:
-            channel (grpc.Channel): The RPC channel to use.
-            filenames (iterable[str]): Optional. Proto file names, with path,
-                to request.
-            symbols (iterable[str]): Optional. Fully qualified symbol names
-                for which to request proto files.
-
-        Raises:
-            ServiceError: Reflection service responded with an error.
-            grpc.RpcError: Lower level RPC error.
-        """
-        return self._engine.load_protocols(channel, filenames, symbols)
-
-    def service_stub_class(self, name):
-        """Get the stub class for the specified service type.
-
-        Args:
-            name: Fully qualified name of the protocol service.
-
-        Raises:
-            KeyError: File defining service has not been loaded.
-        """
-        service = self._engine.pool.FindServiceByName(name)
-        methods = self._engine.methods_by_file[service.file.name][service.name]
-        return self._engine.gen_stub_class(service, methods)[0]
-
-    def message_class(self, name):
-        """Get the class for the specified message type.
-
-        Args:
-            name: Fully qualified name of the protocol message.
-
-        Raises:
-            KeyError: File defining message has not been loaded.
-        """
-        return self._engine.message_from_descr(
-            self._engine.pool.FindMessageTypeByName(name)
-        )
-
-    def enum_class(self, name):
-        """Get the class for the specified enum type.
-
-        Args:
-            name: Fully qualified name of the protocol enum.
-
-        Raises:
-            KeyError: File defining enum has not been loaded.
-        """
-        return enum_from_descr(self._engine.pool.FindEnumTypeByName(name))

splight-lib-3.3.7/splight_lib/component/log_streamer.py

@@ -1,107 +0,0 @@
-import re
-import sys
-from queue import Empty, Queue
-from subprocess import Popen
-from threading import Thread
-from typing import Generator, Optional
-
-from splight_lib.client.grpc.client import LogsGRPCClient, LogsGRPCError
-from splight_lib.component.exceptions import LogsStreamerError
-from splight_lib.settings import settings
-
-LOG_FORMAT = r"^.* \| .* \| .*:\d{2,} \| .* "
-LOG_PATTERN = re.compile(LOG_FORMAT)
-
-
-class ComponentLogsStreamer:
-    def __init__(self, process: Popen, component_id: str):
-        self._process = process
-        self._component_id = component_id
-
-        try:
-            self._client = self._create_client()
-        except Exception as exc:
-            raise LogsStreamerError(
-                "Unable to connect to gRPC server"
-            ) from exc
-        self._logs_entry = self._client._log_entry
-        self._thread: Optional[Thread] = None
-        self._queue: Optional[Queue] = None
-        self._running: bool = False
-
-    def _create_client(self):
-        client = LogsGRPCClient(
-            grpc_host=settings.SPLIGHT_GRPC_HOST, secure_channel=True
-        )
-        client.set_authorization_header(
-            access_id=settings.SPLIGHT_ACCESS_ID,
-            secret_key=settings.SPLIGHT_SECRET_KEY,
-        )
-        return client
-
-    def start(self):
-        self._thread = Thread(target=self._consume_logs, daemon=True)
-        self._queue = Queue()
-        self._running = True
-        self._thread.start()
-        self._run()
-
-    def stop(self):
-        self._running = False
-        self._thread.join(timeout=10)
-        self._queue = None
-        self._thread = None
-
-    def _run(self):
-        while self._running:
-            try:
-                self._client.stream_logs(
-                    self.logs_iterator, self._component_id
-                )
-            except LogsGRPCError as exc:
-                raise LogsStreamerError(
-                    "Component Logs stream stopped"
-                ) from exc
-
-    def _consume_logs(self):
-        reader = iter(self._process.stdout.readline, "")
-        for new_line in reader:
-            if self._process.poll() is not None:
-                output, error = self._process.communicate()
-                output_msg = output.decode("utf-8")
-                error_msg = error.decode("utf-8")
-                self._queue.put(output_msg)
-                self._queue.put(error_msg)
-                self._running = False
-                break
-            self._queue.put(new_line.decode("utf-8"))
-
-    def logs_iterator(self) -> Generator:
-        self._message_buffer = []
-        while True:
-            try:
-                message = self._queue.get(timeout=10)
-            except Empty:
-                msg = "".join(self._message_buffer)
-                if msg:
-                    sys.stdout.write(msg)
-                    yield msg
-                return
-
-            full_msg = self._generate_message(message)
-            if not full_msg:
-                continue
-            yield full_msg
-            sys.stdout.write(full_msg)
-
-    def _generate_message(self, raw_msg: str) -> Optional[str]:
-        if self._is_log(raw_msg):
-            msg = "".join(self._message_buffer)
-            self._message_buffer = [raw_msg]
-            return msg
-        self._message_buffer.append(raw_msg)
-        return None
-
-    def _is_log(self, raw_msg: str) -> bool:
-        match = LOG_PATTERN.match(raw_msg)
-        return True if match else False