splight-lib 5.22.0.dev0__tar.gz → 5.22.2__tar.gz

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/Makefile

@@ -48,17 +48,3 @@ format: install-dev
 
 check-format: install-dev
 	uv run pre-commit run --all-files
-	# --show-diff-on-failure
-
-# format: 
-# 	uv run isort splight_lib/
-# 	uv run ruff format splight_lib/
-#
-# check_isort:
-# 	uv run isort --check-only --diff splight_lib/
-#
-# check_ruff:
-# 	uv run ruff format --check --diff splight_lib/
-#
-# check_format: check_ruff check_isort 
-#

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splight-lib
-Version: 5.22.0.dev0
+Version: 5.22.2
 Summary: Splight Python Library
 Author-email: Splight Dev <dev@splight.com>
 License-File: LICENSE.txt
@@ -86,7 +86,7 @@ the version you should use the command
 ```bash
 make update-version scope=<scope>
 ```
-where scope can be `mahor`, `minor` or `patch`.
+where scope can be `major`, `minor` or `patch`.
 
 ## Tests
 

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/README.md

@@ -50,7 +50,7 @@ the version you should use the command
 ```bash
 make update-version scope=<scope>
 ```
-where scope can be `mahor`, `minor` or `patch`.
+where scope can be `major`, `minor` or `patch`.
 
 ## Tests
 

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splight-lib"
-version = "5.22.0.dev0"
+version = "5.22.2"
 description = "Splight Python Library"
 authors = [
     {name = "Splight Dev",email = "dev@splight.com"}
@@ -68,7 +68,6 @@ profile = "black"
 line_length = 79
 
 [tool.pytest.ini_options]
-# testpaths = ["splight_lib/component/tests/"]
 filterwarnings = [
     "error",
     "ignore::RuntimeWarning",
@@ -115,6 +114,8 @@ indent-width = 4
 
 target-version = "py311"
 
+extend-select = ["F", "I"]
+
 [tool.ruff.format]
 # Like Black, use double quotes for strings.
 quote-style = "double"

splight_lib-5.22.2/splight_lib/client/datalake/__init__.py

@@ -0,0 +1,5 @@
+from splight_lib.client.datalake.builder import DatalakeClientBuilder
+
+__all__ = [
+    DatalakeClientBuilder,
+]

splight_lib-5.22.2/splight_lib/client/datalake/builder.py

@@ -0,0 +1,27 @@
+from typing import Any
+
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.v3.builder import (
+    DatalakeClientBuilder as V3DatalakeClientBuilder,
+)
+from splight_lib.client.datalake.v4.builder import (
+    DatalakeClientBuilder as V4DatalakeClientBuilder,
+)
+from splight_lib.settings import DatalakeClientType, SplightAPIVersion
+
+
+class DatalakeClientBuilder:
+    @staticmethod
+    def build(
+        version: SplightAPIVersion = SplightAPIVersion.V3,
+        dl_client_type: DatalakeClientType = DatalakeClientType.BUFFERED_ASYNC,
+        parameters: dict[str, Any] = {},
+    ) -> AbstractDatalakeClient:
+        if version == SplightAPIVersion.V3:
+            Builder = V3DatalakeClientBuilder
+        elif version == SplightAPIVersion.V4:
+            Builder = V4DatalakeClientBuilder
+        else:
+            raise ValueError(f"Unsupported API version: {version}")
+
+        return Builder.build(dl_client_type, parameters)

{splight_lib-5.22.0.dev0/splight_lib/client/datalake → splight_lib-5.22.2/splight_lib/client/datalake/common}/__init__.py

@@ -1,5 +1,5 @@
-from splight_lib.client.datalake.builder import DatalakeClientBuilder
-from splight_lib.client.datalake.remote_client import (
+from splight_lib.client.datalake.v3.builder import DatalakeClientBuilder
+from splight_lib.client.datalake.v3.remote_client import (
     BufferedAsyncRemoteDatalakeClient,
     BufferedSyncRemoteDataClient,
     SyncRemoteDatalakeClient,

splight_lib-5.22.2/splight_lib/client/datalake/v3/__init__.py

@@ -0,0 +1,13 @@
+from splight_lib.client.datalake.v3.builder import DatalakeClientBuilder
+from splight_lib.client.datalake.v3.remote_client import (
+    BufferedAsyncRemoteDatalakeClient,
+    BufferedSyncRemoteDataClient,
+    SyncRemoteDatalakeClient,
+)
+
+__all__ = [
+    DatalakeClientBuilder,
+    SyncRemoteDatalakeClient,
+    BufferedAsyncRemoteDatalakeClient,
+    BufferedSyncRemoteDataClient,
+]

{splight_lib-5.22.0.dev0/splight_lib/client/datalake → splight_lib-5.22.2/splight_lib/client/datalake/v3}/builder.py

@@ -1,7 +1,7 @@
 from typing import Any
 
-from splight_lib.client.datalake.abstract import AbstractDatalakeClient
-from splight_lib.client.datalake.remote_client import (
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.v3.remote_client import (
     BufferedAsyncRemoteDatalakeClient,
     BufferedSyncRemoteDataClient,
     SyncRemoteDatalakeClient,

{splight_lib-5.22.0.dev0/splight_lib/client/datalake → splight_lib-5.22.2/splight_lib/client/datalake/v3}/remote_client.py

@@ -6,13 +6,13 @@ from httpx import HTTPTransport
 from retry import retry
 
 from splight_lib.auth import SplightAuthToken
-from splight_lib.client.datalake.abstract import (
+from splight_lib.client.datalake.common.abstract import (
     AbstractDatalakeClient,
     Records,
 )
-from splight_lib.client.datalake.buffer import DatalakeDocumentBuffer
-from splight_lib.client.datalake.classmap import COLLECTION_PREFIXS_MAP
-from splight_lib.client.datalake.exceptions import DatalakeRequestError
+from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
+from splight_lib.client.datalake.v3.classmap import COLLECTION_PREFIXS_MAP
+from splight_lib.client.datalake.v3.exceptions import DatalakeRequestError
 from splight_lib.client.exceptions import SPLIGHT_REQUEST_EXCEPTIONS
 from splight_lib.logging._internal import LogTags, get_splight_logger
 from splight_lib.restclient import SplightRestClient

splight_lib-5.22.2/splight_lib/client/datalake/v4/__init__.py

@@ -0,0 +1,13 @@
+from splight_lib.client.datalake.v4.builder import DatalakeClientBuilder
+from splight_lib.client.datalake.v4.remote_client import (
+    BufferedAsyncRemoteDatalakeClient,
+    BufferedSyncRemoteDataClient,
+    SyncRemoteDatalakeClient,
+)
+
+__all__ = [
+    DatalakeClientBuilder,
+    SyncRemoteDatalakeClient,
+    BufferedAsyncRemoteDatalakeClient,
+    BufferedSyncRemoteDataClient,
+]

splight_lib-5.22.2/splight_lib/client/datalake/v4/builder.py

@@ -0,0 +1,24 @@
+from typing import Any
+
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.v4.remote_client import (
+    BufferedAsyncRemoteDatalakeClient,
+    BufferedSyncRemoteDataClient,
+    SyncRemoteDatalakeClient,
+)
+from splight_lib.settings import DatalakeClientType
+
+DL_CLIENT_TYPE_MAP = {
+    DatalakeClientType.BUFFERED_ASYNC: BufferedAsyncRemoteDatalakeClient,
+    DatalakeClientType.BUFFERED_SYNC: BufferedSyncRemoteDataClient,
+    DatalakeClientType.SYNC: SyncRemoteDatalakeClient,
+}
+
+
+class DatalakeClientBuilder:
+    @staticmethod
+    def build(
+        dl_client_type: DatalakeClientType = DatalakeClientType.BUFFERED_ASYNC,
+        parameters: dict[str, Any] = {},
+    ) -> AbstractDatalakeClient:
+        return DL_CLIENT_TYPE_MAP[dl_client_type](**parameters)

splight_lib-5.22.2/splight_lib/client/datalake/v4/exceptions.py

@@ -0,0 +1,12 @@
+class DatalakeRequestError(Exception):
+    def __init__(self, status_code: int, message: str):
+        self._msg = f"Request failed with status code {status_code}: {message}"
+        super().__init__(self._msg)
+
+
+class InvalidCollectionName(Exception):
+    def __init__(self, collection: str):
+        self._msg = f"Collection {collection} is not a valid collection"
+
+    def __str__(self) -> str:
+        return self._msg

splight_lib-5.22.2/splight_lib/client/datalake/v4/remote_client.py

@@ -0,0 +1,258 @@
+from threading import Lock, Thread
+from time import sleep
+
+from furl import furl
+from httpx import HTTPTransport
+from retry import retry
+
+from splight_lib.auth import SplightAuthToken
+from splight_lib.client.datalake.common.abstract import (
+    AbstractDatalakeClient,
+    Records,
+)
+from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
+from splight_lib.client.datalake.v4.exceptions import DatalakeRequestError
+from splight_lib.client.exceptions import SPLIGHT_REQUEST_EXCEPTIONS
+from splight_lib.logging._internal import LogTags, get_splight_logger
+from splight_lib.restclient import SplightRestClient
+from splight_lib.settings import SplightAPIVersion
+
+logger = get_splight_logger()
+
+EXCEPTIONS = (*SPLIGHT_REQUEST_EXCEPTIONS, DatalakeRequestError)
+
+
+class SyncRemoteDatalakeClient(AbstractDatalakeClient):
+    def __init__(
+        self,
+        base_url: str,
+        resource: str,
+        access_id: str,
+        secret_key: str,
+        api_version: SplightAPIVersion = SplightAPIVersion.V4,
+        *args,
+        **kwargs,
+    ):
+        super().__init__()
+        self._base_url = furl(base_url)
+        self.resource = resource
+        token = SplightAuthToken(
+            access_key=access_id,
+            secret_key=secret_key,
+        )
+
+        self._restclient = SplightRestClient(
+            transport=HTTPTransport(retries=3)
+        )
+        self._restclient.update_headers(token.header)
+        logger.debug(
+            "Remote datalake client initialized.", tags=LogTags.DATALAKE
+        )
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def save(self, records: dict) -> list[dict]:
+        url = self._base_url / f"{self.prefix}/write/"
+        response = self._restclient.post(url, json=records)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return records["records"]
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    async def async_save(
+        self,
+        records: dict,
+    ) -> list[dict]:
+        url = self._base_url / f"{self.prefix}/write/"
+        response = await self._restclient.async_post(url, json=records)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return records["records"]
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def _get(self, request: dict) -> list[dict]:
+        url = self._base_url / f"{self.prefix}/read/"
+        response = self._restclient.post(url, json=request)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return response.json()
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    async def _async_get(self, request: dict) -> list[dict]:
+        url = self._base_url / f"{self.prefix}/read/"
+        response = await self._restclient.async_post(url, json=request)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return response.json()
+
+    @property
+    def prefix(self) -> str:
+        return f"v4/data/{self.resource}"
+
+
+class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
+    def __init__(
+        self,
+        base_url: str,
+        access_id: str,
+        secret_key: str,
+        api_version: SplightAPIVersion,
+        buffer_size: int = 500,
+        buffer_timeout: float = 60,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(
+            base_url=base_url,
+            access_id=access_id,
+            secret_key=secret_key,
+            buffer_size=buffer_size,
+            buffer_timeout=buffer_timeout,
+            *args,
+            **kwargs,
+        )
+        self._base_url = furl(base_url)
+
+        token = SplightAuthToken(
+            access_key=access_id,
+            secret_key=secret_key,
+        )
+        self._restclient = SplightRestClient()
+        self._restclient.update_headers(token.header)
+
+        logger.debug(
+            "Initializing buffer with size %s and timeout %s",
+            buffer_size,
+            buffer_timeout,
+        )
+        self._data_buffers = {
+            "default": DatalakeDocumentBuffer(buffer_size, buffer_timeout),
+            "routineEvaluations": DatalakeDocumentBuffer(
+                buffer_size, buffer_timeout
+            ),
+        }
+        self._lock = Lock()
+        self._flush_thread = Thread(target=self._flusher, daemon=True)
+        self._flush_thread.start()
+        logger.debug(
+            "Buffered Remote datalake client initialized.",
+            tags=LogTags.DATALAKE,
+        )
+
+    def save(self, records: dict) -> list[dict]:
+        logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
+        instances = records["records"]
+        buffer = self._data_buffers["default"]
+        with self._lock:
+            if buffer.should_flush():
+                logger.debug(
+                    "Flushing datalake buffer with %s elements",
+                    len(buffer.data),
+                )
+                self._send_documents(buffer.data)
+                buffer.reset()
+            buffer.add_documents(instances)
+        return instances
+
+    def _flusher(self):
+        while True:
+            for _, buffer in self._data_buffers.items():
+                self._flush_buffer(buffer)
+            sleep(0.5)
+
+    def _flush_buffer(self, buffer: DatalakeDocumentBuffer) -> None:
+        with self._lock:
+            if buffer.should_flush():
+                try:
+                    logger.debug(
+                        "Flushing datalake buffer with %s elements",
+                        len(buffer.data),
+                    )
+                    self._send_documents(buffer.data)
+                    buffer.reset()
+                except Exception:
+                    logger.error("Unable to save documents", exc_info=True)
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def _send_documents(self, docs: list[dict]) -> list[dict]:
+        url = self._base_url / f"{self.prefix}/write/"
+        data = {
+            "records": docs,
+        }
+        response = self._restclient.post(url, json=data)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return docs
+
+
+class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
+    def __init__(
+        self,
+        base_url: str,
+        access_id: str,
+        secret_key: str,
+        api_version: SplightAPIVersion,
+        buffer_size: int = 500,
+        buffer_timeout: float = 60,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(
+            base_url=base_url,
+            access_id=access_id,
+            secret_key=secret_key,
+            buffer_size=buffer_size,
+            buffer_timeout=buffer_timeout,
+            *args,
+            **kwargs,
+        )
+        self._base_url = furl(base_url)
+        token = SplightAuthToken(
+            access_key=access_id,
+            secret_key=secret_key,
+        )
+        self._restclient = SplightRestClient()
+        self._restclient.update_headers(token.header)
+
+        logger.debug(
+            "Initializing buffer with size %s and timeout %s",
+            buffer_size,
+            buffer_timeout,
+            tags=LogTags.DATALAKE,
+        )
+        self._data_buffers = {
+            "default": DatalakeDocumentBuffer(buffer_size, buffer_timeout),
+            "routine_evaluations": DatalakeDocumentBuffer(
+                buffer_size, buffer_timeout
+            ),
+        }
+        self._lock = Lock()
+        logger.debug(
+            "Synchronous Buffered Remote datalake client initialized.",
+            tags=LogTags.DATALAKE,
+        )
+
+    def save(self, records: Records) -> list[dict]:
+        logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
+        buffer = self._data_buffers["default"]
+        with self._lock:
+            buffer.add_documents(records["records"])
+            if buffer.should_flush():
+                logger.debug(
+                    "Flushing datalake buffer with %s elements",
+                    len(buffer.data),
+                    tags=LogTags.DATALAKE,
+                )
+                self._send_documents(buffer.data)
+                buffer.reset()
+        return records["records"]
+
+    @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
+    def _send_documents(self, docs: list[dict]) -> list[dict]:
+        url = self._base_url / f"{self.prefix}/write/"
+        data = {
+            "records": docs,
+        }
+        response = self._restclient.post(url, json=data)
+        if response.is_error:
+            raise DatalakeRequestError(response.status_code, response.text)
+        return docs

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/splight_lib/client/tests/test_datalake.py

@@ -2,10 +2,10 @@ import os
 
 from pytest_mock import MockerFixture
 
-from splight_lib.client.datalake import SyncRemoteDatalakeClient  # noqa E402
-from splight_lib.client.datalake.remote_client import (  # noqa E402
-    SplightRestClient,
+from splight_lib.client.datalake.v3 import (  # noqa E402
+    SyncRemoteDatalakeClient,
 )
+from splight_lib.restclient import SplightRestClient
 
 base_url = "http://test.com"
 os.environ["ACCESS_ID"] = "access_id"

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/splight_lib/conftest.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import pytest
 
 from splight_lib.client.database.abstract import AbstractDatabaseClient
-from splight_lib.client.datalake.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
 from splight_lib.settings import SplightAPIVersion, api_settings
 
 # Only load these tests when API_VERSION is v4

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/splight_lib/models/__init__.py

@@ -169,6 +169,7 @@ elif api_version == SplightAPIVersion.V4:
         AssetKind,
         AssetRelationship,
         Attribute,
+        AttributeDocument,
         AttributeType,
         Battery,
         Boolean,
@@ -179,7 +180,6 @@ elif api_version == SplightAPIVersion.V4:
         ComponentType,
         CustomType,
         DataAddress,
-        DataRequest,
         Endpoint,
         ExternalGrid,
         File,
@@ -195,8 +195,9 @@ elif api_version == SplightAPIVersion.V4:
         Number,
         Output,
         Parameter,
-        PipelineStep,
         PrivacyPolicy,
+        Query,
+        Records,
         ResourceSummary,
         Routine,
         RoutineEvaluation,
@@ -206,10 +207,10 @@ elif api_version == SplightAPIVersion.V4:
         Segment,
         Server,
         SlackLine,
+        SolutionOutputDocument,
         SplightDatalakeBaseModel,
         String,
         Tag,
-        Trace,
         Transformer,
         ValueType,
         get_field_value,
@@ -235,9 +236,10 @@ elif api_version == SplightAPIVersion.V4:
         "Parameter",
         "DB_MODEL_TYPE_MAPPING",
         "SplightDatalakeBaseModel",
-        "DataRequest",
-        "PipelineStep",
-        "Trace",
+        "Query",
+        "Records",
+        "AttributeDocument",
+        "SolutionOutputDocument",
         "File",
         "DataAddress",
         "InputDataAddress",

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/splight_lib/models/_v3/datalake.py

@@ -6,8 +6,8 @@ from typing import Annotated, Any, Generator, Generic, Literal, Self, TypeVar
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 
 from splight_lib.client.datalake import DatalakeClientBuilder
-from splight_lib.client.datalake.abstract import AbstractDatalakeClient
-from splight_lib.client.datalake.constants import StepName
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.v3.constants import StepName
 from splight_lib.models._v3.asset import Asset
 from splight_lib.models._v3.attribute import Attribute
 from splight_lib.models._v3.exceptions import TraceAlreadyExistsError
@@ -27,8 +27,10 @@ def hash(string: str) -> str:
 
 def get_datalake_client() -> AbstractDatalakeClient:
     return DatalakeClientBuilder.build(
-        dl_client_type=datalake_settings.DL_CLIENT_TYPE,
+        version=api_settings.API_VERSION,
+        dl_client_type="sync",
         parameters={
+            "resource": "attributes",
             "base_url": workspace_settings.SPLIGHT_PLATFORM_API_HOST,
             "access_id": workspace_settings.SPLIGHT_ACCESS_ID,
             "secret_key": workspace_settings.SPLIGHT_SECRET_KEY,

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/splight_lib/models/_v3/datalake_base.py

@@ -61,7 +61,7 @@ class SplightDatalakeBaseModel(BaseModel):
             cls, asset, attribute, extra_pipeline, **params
         )
         instances = request.apply()
-        df = pd.DataFrame([instance.dict() for instance in instances])
+        df = pd.DataFrame([instance.model_dump() for instance in instances])
         if not df.empty:
             df.index = df["timestamp"]
             df.drop(columns="timestamp", inplace=True)

{splight_lib-5.22.0.dev0 → splight_lib-5.22.2}/splight_lib/models/_v4/__init__.py

@@ -27,7 +27,12 @@ from splight_lib.models._v4.component import (
     get_field_value,
 )
 from splight_lib.models._v4.data_address import DataAddresses as DataAddress
-from splight_lib.models._v4.datalake import DataRequest, PipelineStep, Trace
+from splight_lib.models._v4.datalake import (
+    AttributeDocument,
+    Query,
+    Records,
+    SolutionOutputDocument,
+)
 from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.external_grid import ExternalGrid
 from splight_lib.models._v4.file import File
@@ -73,7 +78,10 @@ __all__ = [
     "get_field_value",
     "Parameter",
     "DB_MODEL_TYPE_MAPPING",
-    "DataRequest",
+    "Query",
+    "Records",
+    "AttributeDocument",
+    "SolutionOutputDocument",
     "DataAddress",
     "InputDataAddress",
     "PipelineStep",