splight-lib 4.0.0.dev5__tar.gz → 4.1.0.dev1__tar.gz

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: splight-lib
-Version: 4.0.0.dev5
+Version: 4.1.0.dev1
 Summary: Library for public use. Splight
 Home-page: UNKNOWN
 Author: Splight

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/setup.py

@@ -14,7 +14,7 @@ test_requires = [
 
 setup(
     name="splight-lib",
-    version="4.0.0.dev5",
+    version="4.1.0.dev1",
     author="Splight",
     author_email="factory@splight-ae.com",
     packages=find_packages(),

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/client/datalake/abstract.py

@@ -13,12 +13,22 @@ class AbstractDatalakeClient(AbstractClient):
         kwargs["count_func"] = "None"
         return QuerySet(self, *args, **kwargs)
 
+    async def async_get(self, *args, **kwargs):
+        # TODO: consider using an async QuerySet
+        return await self._async_raw_get(*args, **kwargs)
+
     @abstractmethod
     def save(
         self, collection: str, instances: Union[List[Dict], Dict]
     ) -> List[dict]:
         pass
 
+    @abstractmethod
+    async def async_save(
+        self, collection: str, instances: Union[List[Dict], Dict]
+    ) -> List[dict]:
+        pass
+
     @abstractmethod
     def delete(self, collection: str, **kwargs) -> None:
         pass
@@ -65,6 +75,21 @@ class AbstractDatalakeClient(AbstractClient):
     ) -> List[Dict]:
         pass
 
+    @abstractmethod
+    async def _async_raw_get(
+        self,
+        resource_name: str,
+        collection: str,
+        limit_: int = 50,
+        skip_: int = 0,
+        sort: Union[List, str] = ["timestamp__desc"],
+        group_id: Optional[Union[List, str]] = None,
+        group_fields: Optional[Union[List, str]] = None,
+        tzinfo: timezone = timezone(timedelta()),
+        **filters,
+    ) -> List[Dict]:
+        pass
+
     @abstractmethod
     def execute_query(
         self,

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/client/datalake/remote_client.py

@@ -49,6 +49,20 @@ class RemoteDatalakeClient(AbstractDatalakeClient, AbstractRemoteClient):
         response.raise_for_status()
         return instances
 
+    @retry(SPLIGHT_REQUEST_EXCEPTIONS, tries=3, delay=2, jitter=1)
+    async def async_save(
+        self,
+        collection: str,
+        instances: Union[List[Dict], Dict],
+    ) -> List[dict]:
+        instances = instances if isinstance(instances, list) else [instances]
+        url = self._base_url / f"{self._PREFIX}/save/"
+        response = await self._restclient.async_post(
+            url, params={"source": collection}, json=instances
+        )
+        response.raise_for_status()
+        return instances
+
     @retry(SPLIGHT_REQUEST_EXCEPTIONS, tries=3, delay=2, jitter=1)
     def _raw_get(
         self,
@@ -85,6 +99,42 @@ class RemoteDatalakeClient(AbstractDatalakeClient, AbstractRemoteClient):
 
         return output
 
+    @retry(SPLIGHT_REQUEST_EXCEPTIONS, tries=3, delay=2, jitter=1)
+    async def _async_raw_get(
+        self,
+        resource_name: str,
+        collection: str,
+        limit_: int = 50,
+        skip_: int = 0,
+        sort: Union[List, str] = ["timestamp__desc"],
+        group_id: Optional[Union[List, str]] = None,
+        group_fields: Optional[Union[List, str]] = None,
+        tzinfo: timezone = timezone(timedelta()),
+        **filters,
+    ) -> List[Dict]:
+        # GET /datalake/data/
+        url = self._base_url / f"{self._PREFIX}/data/"
+
+        filters.update(
+            {
+                "source": collection,
+                "output_format": resource_name,
+                "sort": sort,
+                "limit_": limit_,
+                "skip_": skip_,
+                "group_id": group_id,
+                "group_fields": group_fields,
+                # "tzinfo": tzinfo
+            }
+        )
+
+        params = self._parse_params(**filters)
+        response = await self._restclient.async_get(url, params=params)
+        response.raise_for_status()
+        output = response.json()["results"]
+
+        return output
+
     @retry(SPLIGHT_REQUEST_EXCEPTIONS, tries=3, delay=2, jitter=1)
     def delete(self, collection: str, **kwargs) -> None:
         # DELETE /datalake/delete/

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/component/abstract.py

@@ -1,16 +1,13 @@
 import os
 import sys
-import traceback
+from abc import ABC, abstractmethod
 from functools import partial
 from tempfile import NamedTemporaryFile
 from time import sleep
-from typing import Dict, List, Optional, Type
+from typing import Callable, Dict, List, Optional, Type
 
-from furl import furl
 from pydantic import BaseModel, create_model
-from retry import retry
 
-from splight_lib.auth import SplightAuthToken
 from splight_lib.client.communication import RemoteCommunicationClient
 from splight_lib.communication.event_handler import (
     command_event_handler,
@@ -18,7 +15,6 @@ from splight_lib.communication.event_handler import (
     setpoint_event_handler,
 )
 from splight_lib.component.exceptions import (
-    DuplicatedComponentException,
     InvalidBidingObject,
     MissingBindingCallback,
     MissingCommandCallback,
@@ -41,12 +37,7 @@ from splight_lib.models.component import (
 )
 from splight_lib.models.event import EventNames
 from splight_lib.models.setpoint import SetPoint
-from splight_lib.restclient import (
-    ConnectError,
-    HTTPError,
-    SplightRestClient,
-    Timeout,
-)
+from splight_lib.restclient import ConnectError, HTTPError, Timeout
 from splight_lib.settings import settings
 
 REQUEST_EXCEPTIONS = (ConnectError, HTTPError, Timeout)
@@ -83,16 +74,14 @@ class HealthCheckProcessor:
             if not is_alive:
                 exc = self._engine.get_last_exception()
                 self._log_exception(exc)
-                self._logger.error(
-                    "Healthcheck task failed.", tags=LogTags.RUNTIME
-                )
+                self._logger.info("Healthcheck finished", tags=LogTags.RUNTIME)
                 self._health_file.close()
-                self._logger.error(
+                self._logger.info(
                     "Healthcheck file removed: %s",
                     self._health_file,
                     tags=LogTags.RUNTIME,
                 )
-                sys.exit(1)
+                break
             sleep(self._HEALTHCHECK_INTERVAL)
 
     def stop(self):
@@ -106,7 +95,7 @@ class HealthCheckProcessor:
             self._logger.exception(exc, exc_info=(exc_type, exc, stack))
 
 
-class SplightBaseComponent:
+class SplightBaseComponent(ABC):
     def __init__(
         self,
         component_id: Optional[str] = None,
@@ -121,9 +110,9 @@ class SplightBaseComponent:
             instance_id=component_id,
         )
         self._execution_engine = ExecutionClient()
-        health_check = HealthCheckProcessor(self._execution_engine)
+        self._health_check = HealthCheckProcessor(self._execution_engine)
         self._health_check_thread = Thread(
-            target=health_check.start, args=(), daemon=False
+            target=self._health_check.start, args=(), daemon=False
         )
         # We can't add the healthcheck thread into the execution engine
         # because that thread should stop if any of the registered threads
@@ -164,6 +153,8 @@ class SplightBaseComponent:
         self._custom_types = self._get_custom_type_model(component_objects)
         self._routines = self._get_routine_model(routines_objects)
 
+        self.start = self._wrap_start(self.start)
+
     @property
     def input(self) -> BaseModel:
         return self._input
@@ -184,6 +175,36 @@ class SplightBaseComponent:
     def execution_engine(self) -> ExecutionClient:
         return self._execution_engine
 
+    def _register_exit(self):
+        if self._execution_engine.get_last_exception():
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
+    def _wrap_start(self, original_start: Callable) -> Callable:
+        """Wraps the start method to wait for all threads to finish.
+
+        Parameters
+        ----------
+        original_start: Callable
+            The implemented start method from the component
+
+        Returns
+        -------
+        Callable
+            The start method wrapped
+        """
+
+        def wrapper():
+            original_start()
+            for thread in self._execution_engine.threads:
+                thread.join()
+
+            self._health_check_thread.join()
+            self._register_exit()
+
+        return wrapper
+
     def _get_custom_type_model(
         self, component_object: Dict[str, Type[ComponentObjectInstance]]
     ) -> BaseModel:
@@ -299,10 +320,10 @@ class SplightBaseComponent:
                 ),
             )
 
+    @abstractmethod
     def start(self):
         raise NotImplementedError()
 
     def stop(self):
-        self._execution_engine.stop(self._health_check_thread)
         self._execution_engine.terminate_all()
         sys.exit(1)

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/execution.py

@@ -1,4 +1,3 @@
-import atexit
 import sys
 import threading
 import time
@@ -153,17 +152,20 @@ class TaskMap:
 
 
 class Scheduler:
-    def __init__(self, *args, **kwargs) -> None:
+    def __init__(self, event: Event) -> None:
+        # The _event attr is used for controling the scheduler loop meanwhile
+        # the _tasks_event is used to notify the scheduler that a task should
+        # be executed
+        self._event = event
         self._tasks = TaskMap()
-        self._event = Event()
+        self._tasks_event = Event()
         self._mutex = Lock()
         self._to_add: List[Task] = []
         self._to_remove: List[Task] = []
 
     def start(self) -> None:
         """Scheduler infinite loop."""
-        self._stop = False
-        while not self._stop:
+        while self._event.is_set():
             # Update the task list
             self._update_task_list()
             # Run the scheduler task
@@ -172,11 +174,12 @@ class Scheduler:
             next_event_time = near_event - time.time()
 
             # Wait until next event or someone trigger the event
-            self._event.wait(timeout=next_event_time)
-            self._event.clear()
+            self._tasks_event.wait(timeout=next_event_time)
+            self._tasks_event.clear()
 
     def stop(self) -> None:
-        self._stop = True
+        if self._event.is_set():
+            self._event.clear()
 
     def _schedule(self) -> float:
         """Scheduler main task."""
@@ -197,7 +200,7 @@ class Scheduler:
         Unlock the scheduler if is locked.
         This will run the scheduler main task.
         """
-        self._event.set()
+        self._tasks_event.set()
 
     def schedule(self, task: Task) -> None:
         """
@@ -242,20 +245,8 @@ class Thread(DefaultThread):
     def __init__(self, target: Callable, args: Tuple = (), **kwargs) -> None:
         target = self.store_result(target)
         self.result = Empty()
-        self._exc = None
         super().__init__(target=target, args=args, name=target, **kwargs)
 
-    @property
-    def exc(self):
-        return self._exc
-
-    def run(self):
-        try:
-            super().run()
-        except Exception as exc:
-            self._exc = exc
-            raise exc
-
     def store_result(self, func: Callable) -> Callable:
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -288,31 +279,56 @@ class Popen(DefaultPopen):
 
 class ExecutionClient(AbstractClient):
     def __init__(self, *args, **kwargs):
+        self._event = Event()
         self.processes: List[Popen] = []
         self.threads: List[Thread] = []
 
         self._register_exit_functions()
         super(ExecutionClient, self).__init__(*args, **kwargs)
         logger.info("Execution client initialized.", tags=LogTags.RUNTIME)
+        self._exc = None
+        self._thread_exc = None
+
+    @property
+    def event(self) -> Event:
+        return self._event
 
     def _register_exit_functions(self) -> None:
         excepthook = sys.excepthook
+        thread_exchook = threading.excepthook
 
         def wrap_excepthook(type, value, traceback):
+            self._exc = (type, value, traceback)
             self.terminate_all()
             excepthook(type, value, traceback)
 
-        atexit.register(self.terminate_all)
+        def wrap_thread_excepthook(args):
+            self._thread_exc = (
+                args.exc_type,
+                args.exc_value,
+                args.exc_traceback,
+            )
+            self.terminate_all()
+            thread_exchook(args)
+
         sys.excepthook = wrap_excepthook
+        threading.excepthook = wrap_thread_excepthook
 
     def __del__(self) -> None:
         self.terminate_all()
 
     def terminate_all(self) -> None:
+        # Set the event to False to stop all threads
+        # This will also stop the scheduler
+        self._event.clear()
+
         for p in self.processes:
             p.terminate()
 
     def start(self, job=Union[Popen, Thread, Task]):
+        # Set the event in true so the threads can run
+        if not self._event.is_set():
+            self._event.set()
         logger.info("Executing new job.", tags=LogTags.RUNTIME)
         if isinstance(job, Popen):
             return self._start_process(job)
@@ -344,7 +360,7 @@ class ExecutionClient(AbstractClient):
         logger.debug("Starting Task", tags=LogTags.RUNTIME)
         if not getattr(self, "_scheduler", None):
             # Instantiate and start Scheduler thread
-            self._scheduler = Scheduler()
+            self._scheduler = Scheduler(self._event)
             self._start_thread(
                 Thread(target=self._scheduler.start, daemon=False)
             )
@@ -356,19 +372,8 @@ class ExecutionClient(AbstractClient):
         return self._scheduler.unschedule(job)
 
     def is_alive(self) -> bool:
-        # We need to also check the main thread just in case some component is
-        # not using the execution client explicitly and is doing some
-        # processing directly in the main thread so the healtcheck should
-        # say the component is healthy.
-        main_thread = threading.main_thread()
-        threads_status = [
-            p.is_alive() or p.exit_ok() for p in self.processes + self.threads
-        ]
-        return (
-            main_thread.is_alive() or all(threads_status)
-            if threads_status
-            else main_thread.is_alive()
-        )
+        threads_status = [p.is_alive() for p in self.processes + self.threads]
+        return all(threads_status)
 
     def healthcheck(self) -> Tuple[bool, ComponentStatus]:
         """Check if the component is alive and return the status.
@@ -393,9 +398,10 @@ class ExecutionClient(AbstractClient):
     def get_last_exception(self) -> Optional[Exception]:
         """Get the last exception thrown in one of the threads.
         It assumes that there is only one thread that crashed
-        Also, only works for the thread not the processes.
+        It only works for the thread not the processes.
         """
-        broken_thread = [x.exc for x in self.threads if x.exc]
-        if broken_thread:
-            return broken_thread[0]
+        if self._exc:
+            return self._exc[1]
+        if self._thread_exc:
+            return self._thread_exc[1]
         return None

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/models/asset.py

@@ -4,6 +4,7 @@ from geojson_pydantic import GeometryCollection
 
 from splight_lib.models.attribute import Attribute
 from splight_lib.models.base import SplightDatabaseBaseModel
+from splight_lib.models.metadata import Metadata
 
 
 class Asset(SplightDatabaseBaseModel):
@@ -12,6 +13,7 @@ class Asset(SplightDatabaseBaseModel):
     description: Optional[str] = None
     tags: List[str] = []
     attributes: List[Attribute] = []
+    metadata: List[Metadata] = []
     verified: bool = False
     geometry: Optional[GeometryCollection]
     centroid_coordinates: Optional[Tuple[float, float]]

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/models/base.py

@@ -94,6 +94,19 @@ class SplightDatalakeBaseModel(BaseModel):
         instances = [cls.parse_obj(item) for item in instances]
         return instances
 
+    @classmethod
+    async def async_get(
+        cls, **params: Dict
+    ) -> List["SplightDatalakeBaseModel"]:
+        dl_client = cls.__get_datalake_client()
+        instances = await dl_client.async_get(
+            resource_name=cls.__name__,
+            collection=cls._collection_name,
+            **params,
+        )
+        instances = [cls.parse_obj(item) for item in instances]
+        return instances
+
     @classmethod
     def get_dataframe(cls, **params: Dict) -> pd.DataFrame:
         dl_client = cls.__get_datalake_client()
@@ -111,6 +124,14 @@ class SplightDatalakeBaseModel(BaseModel):
             instances=json.loads(self.json()),
         )
 
+    async def async_save(self):
+        dl_client = self.__get_datalake_client()
+
+        await dl_client.async_save(
+            collection=self._collection_name,
+            instances=json.loads(self.json()),
+        )
+
     @classmethod
     def save_dataframe(cls, dataframe: pd.DataFrame):
         dl_client = cls.__get_datalake_client()

splight-lib-4.1.0.dev1/splight_lib/models/metadata.py

@@ -0,0 +1,20 @@
+from enum import auto
+from typing import Optional
+
+from strenum import PascalCaseStrEnum
+
+from splight_lib.models.base import SplightDatabaseBaseModel
+
+
+class MetadataType(PascalCaseStrEnum):
+    NUMBER = auto()
+    BOOLEAN = auto()
+    STRING = auto()
+
+
+class Metadata(SplightDatabaseBaseModel):
+    id: Optional[str]
+    name: Optional[str]
+    asset: Optional[str]
+    type: MetadataType = MetadataType.NUMBER
+    value: Optional[str]

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/models/native.py

@@ -24,6 +24,11 @@ class NativeOutput(SplightDatalakeBaseModel):
         params["output_format"] = cls._output_format
         return super().get(**params)
 
+    @classmethod
+    async def async_get(cls, **params: Dict) -> List["NativeOutput"]:
+        params["output_format"] = cls._output_format
+        return await super().async_get(**params)
+
     @classmethod
     def get_dataframe(cls, **params: Dict) -> pd.DataFrame:
         params["output_format"] = cls._output_format

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib/restclient/client.py

@@ -183,9 +183,33 @@ class SplightRestClient:
             default_encoding=default_encoding,
             event_hooks=event_hooks,
         )
+        self._async_client = httpx.AsyncClient(
+            auth=auth,
+            params=params,
+            headers=headers,
+            cookies=cookies,
+            timeout=timeout,
+            verify=verify,
+            cert=cert,
+            http1=http1,
+            http2=http2,
+            proxies=proxies,
+            follow_redirects=allow_redirects,
+            base_url=base_url,
+            limits=limits,
+            max_redirects=max_redirects,
+            transport=transport,
+            app=app,
+            trust_env=trust_env,
+            default_encoding=default_encoding,
+            event_hooks=event_hooks,
+        )
 
     def update_headers(self, new_headers: HeaderTypes):
         self._client.headers = self._client._merge_headers(new_headers)
+        self._async_client.headers = self._async_client._merge_headers(
+            new_headers
+        )
 
     def get(
         self,
@@ -214,6 +238,33 @@ class SplightRestClient:
         )
         return SplightResponse.from_response(raw_response)
 
+    async def async_get(
+        self,
+        url: URLTypes,
+        *,
+        params: Optional[QueryParamTypes] = None,
+        headers: Optional[HeaderTypes] = None,
+        cookies: Optional[CookieTypes] = None,
+        auth: Union[AuthTypes, DefaultClient] = DEFAULT_CLIENT,
+        allow_redirects: Union[bool, DefaultClient] = DEFAULT_CLIENT,
+        timeout: Union[TimeoutTypes, DefaultClient] = DEFAULT_CLIENT,
+    ) -> SplightResponse:
+        """Send a GET request to the specified URL.
+
+        Parameters: See class docstring.
+        """
+        raw_response = await self._async_client.request(
+            self._GET_METHOD,
+            str(url),
+            params=params,
+            headers=headers,
+            cookies=cookies,
+            auth=auth,
+            follow_redirects=allow_redirects,
+            timeout=timeout,
+        )
+        return SplightResponse.from_response(raw_response)
+
     def options(
         self,
         url: URLTypes,
@@ -301,6 +352,39 @@ class SplightRestClient:
         )
         return SplightResponse.from_response(raw_response)
 
+    async def async_post(
+        self,
+        url: URLTypes,
+        *,
+        data: Optional[RequestData] = None,
+        files: Optional[RequestFiles] = None,
+        json: Optional[Any] = None,
+        params: Optional[QueryParamTypes] = None,
+        headers: Optional[HeaderTypes] = None,
+        cookies: Optional[CookieTypes] = None,
+        auth: Union[AuthTypes, DefaultClient] = DEFAULT_CLIENT,
+        allow_redirects: Union[bool, DefaultClient] = DEFAULT_CLIENT,
+        timeout: Union[TimeoutTypes, DefaultClient] = DEFAULT_CLIENT,
+    ) -> SplightResponse:
+        """Send a POST request to the specified URL.
+
+        Parameters: See class docstring.
+        """
+        raw_response = await self._async_client.request(
+            self._POST_METHOD,
+            str(url),
+            data=data,
+            files=files,
+            json=json,
+            params=params,
+            headers=headers,
+            cookies=cookies,
+            auth=auth,
+            follow_redirects=allow_redirects,
+            timeout=timeout,
+        )
+        return SplightResponse.from_response(raw_response)
+
     def put(
         self,
         url: URLTypes,

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: splight-lib
-Version: 4.0.0.dev5
+Version: 4.1.0.dev1
 Summary: Library for public use. Splight
 Home-page: UNKNOWN
 Author: Splight

{splight-lib-4.0.0.dev5 → splight-lib-4.1.0.dev1}/splight_lib.egg-info/SOURCES.txt

@@ -69,6 +69,7 @@ splight_lib/models/event.py
 splight_lib/models/exceptions.py
 splight_lib/models/file.py
 splight_lib/models/hub.py
+splight_lib/models/metadata.py
 splight_lib/models/native.py
 splight_lib/models/pipeline.py
 splight_lib/models/query.py