splight-lib 5.23.3__tar.gz → 5.24.0.dev1__tar.gz

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splight-lib
-Version: 5.23.3
+Version: 5.24.0.dev1
 Summary: Splight Python Library
 Author-email: Splight Dev <dev@splight.com>
 License-File: LICENSE.txt

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splight-lib"
-version = "5.23.3"
+version = "5.24.0.dev1"
 description = "Splight Python Library"
 authors = [
     {name = "Splight Dev",email = "dev@splight.com"}

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/common/abstract.py

@@ -5,7 +5,7 @@ from splight_lib.abstract.client import AbstractRemoteClient, QuerySet
 
 
 class Records(TypedDict):
-    collection: str
+    schema_name: str
     records: list[dict[str, Any]]
 
 

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/v4/builder.py

@@ -6,7 +6,12 @@ from splight_lib.client.datalake.v4.remote_client import (
     BufferedSyncRemoteDataClient,
     SyncRemoteDatalakeClient,
 )
-from splight_lib.settings import DatalakeClientType
+from splight_lib.settings import (
+    DatalakeClientType,
+    SplightAPIVersion,
+    datalake_settings,
+    workspace_settings,
+)
 
 DL_CLIENT_TYPE_MAP = {
     DatalakeClientType.BUFFERED_ASYNC: BufferedAsyncRemoteDatalakeClient,
@@ -22,3 +27,17 @@ class DatalakeClientBuilder:
         parameters: dict[str, Any] = {},
     ) -> AbstractDatalakeClient:
         return DL_CLIENT_TYPE_MAP[dl_client_type](**parameters)
+
+
+def get_datalake_client() -> AbstractDatalakeClient:
+    return DatalakeClientBuilder.build(
+        dl_client_type=datalake_settings.DL_CLIENT_TYPE,
+        parameters={
+            "base_url": workspace_settings.SPLIGHT_PLATFORM_API_HOST,
+            "access_id": workspace_settings.SPLIGHT_ACCESS_ID,
+            "secret_key": workspace_settings.SPLIGHT_SECRET_KEY,
+            "api_version": SplightAPIVersion.V4,
+            "buffer_size": datalake_settings.DL_BUFFER_SIZE,
+            "buffer_timeout": datalake_settings.DL_BUFFER_TIMEOUT,
+        },
+    )

splight_lib-5.24.0.dev1/splight_lib/client/datalake/v4/exceptions.py

@@ -0,0 +1,4 @@
+class DatalakeRequestError(Exception):
+    def __init__(self, status_code: int, message: str):
+        self._msg = f"Request failed with status code {status_code}: {message}"
+        super().__init__(self._msg)

splight_lib-5.24.0.dev1/splight_lib/client/datalake/v4/generic.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+from enum import Enum
+from typing import TypeAlias
+
+Timestamp: TypeAlias = datetime
+Value: TypeAlias = float | bool | str | int
+
+
+class AggregationFunction(str, Enum):
+    SUM = "sum"
+    AVG = "avg"
+    MIN = "min"
+    MAX = "max"
+    COUNT = "count"
+
+
+class TimeUnit(str, Enum):
+    SECOND = "second"
+    MINUTE = "minute"
+    HOUR = "hour"
+    DAY = "day"
+
+
+class TransitionSchemaName(str, Enum):
+    DEFAULT = "default"
+    SOLUTIONS = "solutions"
+
+
+class TransitionSort(int, Enum):
+    ASC = 1
+    DESC = -1

splight_lib-5.24.0.dev1/splight_lib/client/datalake/v4/models.py

@@ -0,0 +1,78 @@
+from typing import Annotated, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from splight_lib.client.datalake.v4.generic import (
+    AggregationFunction,
+    Timestamp,
+    TimeUnit,
+    TransitionSchemaName,
+    TransitionSort,
+    Value,
+)
+
+
+class DefaultEntryKey(BaseModel):
+    asset: str
+    attribute: str
+
+
+class DefaultKeys(BaseModel):
+    schema_name: Literal[TransitionSchemaName.DEFAULT]
+    entries: list[DefaultEntryKey]
+
+    @classmethod
+    def load(cls, entries: list) -> "DefaultKeys":
+        return cls(
+            schema_name=TransitionSchemaName.DEFAULT,
+            entries=entries,
+        )
+
+
+class SolutionEntryKey(BaseModel):
+    solution: str
+    asset: str
+    output: str
+
+
+class SolutionKeys(BaseModel):
+    schema_name: Literal[TransitionSchemaName.SOLUTIONS]
+    entries: list[SolutionEntryKey]
+
+    @classmethod
+    def load(cls, entries: list) -> "SolutionKeys":
+        return cls(
+            schema_name=TransitionSchemaName.SOLUTIONS,
+            entries=entries,
+        )
+
+
+QueryKeys = Annotated[
+    DefaultKeys | SolutionKeys, Field(discriminator="schema_name")
+]
+
+
+class DataReadRequest(BaseModel):
+    keys: QueryKeys
+    start: Optional[Timestamp]
+    end: Optional[Timestamp] = None
+    time_window_unit: TimeUnit = TimeUnit.SECOND
+    time_window_size: int = 1
+    aggregation: AggregationFunction = AggregationFunction.MAX
+    sort: TransitionSort = TransitionSort.DESC
+    limit: int = 1000
+
+
+class DefaultRecord(DefaultEntryKey):
+    value: Value
+    timestamp: Timestamp
+
+
+class SolutionRecord(SolutionEntryKey):
+    value: Value
+    timestamp: Timestamp
+
+
+class DataWriteRequest(BaseModel):
+    schema_name: TransitionSchemaName
+    records: list[SolutionRecord] | list[DefaultRecord]

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/v4/remote_client.py

@@ -12,6 +12,9 @@ from splight_lib.client.datalake.common.abstract import (
 )
 from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
 from splight_lib.client.datalake.v4.exceptions import DatalakeRequestError
+from splight_lib.client.datalake.v4.models import (
+    DataReadRequest,
+)
 from splight_lib.client.exceptions import SPLIGHT_REQUEST_EXCEPTIONS
 from splight_lib.logging._internal import LogTags, get_splight_logger
 from splight_lib.restclient import SplightRestClient
@@ -67,7 +70,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         return records["records"]
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def _get(self, request: dict) -> list[dict]:
+    def _get(self, request: DataReadRequest) -> list[dict]:
         url = self._base_url / f"{self.prefix}/read/"
         response = self._restclient.post(url, json=request)
         if response.is_error:
@@ -75,7 +78,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         return response.json()
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    async def _async_get(self, request: dict) -> list[dict]:
+    async def _async_get(self, request: DataReadRequest) -> list[dict]:
         url = self._base_url / f"{self.prefix}/read/"
         response = await self._restclient.async_post(url, json=request)
         if response.is_error:
@@ -84,8 +87,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
 
     @property
     def prefix(self) -> str:
-        return "v4/data"
-        # return f"v4/data/{self.resource}"
+        return "v4/data/transition"
 
 
 class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
@@ -141,27 +143,27 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
     def save(self, records: dict) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
         instances = records["records"]
-        collection = records["collection"]
-        buffer = self._data_buffers[collection]
+        schema_name = records["schema_name"]
+        buffer = self._data_buffers[schema_name]
         with self._lock:
             if buffer.should_flush():
                 logger.debug(
                     "Flushing datalake buffer with %s elements",
                     len(buffer.data),
                 )
-                self._send_documents(collection, buffer.data)
+                self._send_documents(schema_name, buffer.data)
                 buffer.reset()
             buffer.add_documents(instances)
         return instances
 
     def _flusher(self):
         while True:
-            for collection, buffer in self._data_buffers.items():
-                self._flush_buffer(collection, buffer)
+            for schema_name, buffer in self._data_buffers.items():
+                self._flush_buffer(schema_name, buffer)
             sleep(0.5)
 
     def _flush_buffer(
-        self, collection: str, buffer: DatalakeDocumentBuffer
+        self, schema_name: str, buffer: DatalakeDocumentBuffer
     ) -> None:
         with self._lock:
             if buffer.should_flush():
@@ -170,16 +172,18 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
                         "Flushing datalake buffer with %s elements",
                         len(buffer.data),
                     )
-                    self._send_documents(collection, buffer.data)
+                    self._send_documents(schema_name, buffer.data)
                     buffer.reset()
                 except Exception:
                     logger.error("Unable to save documents", exc_info=True)
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def _send_documents(self, collection: str, docs: list[dict]) -> list[dict]:
+    def _send_documents(
+        self, schema_name: str, docs: list[dict]
+    ) -> list[dict]:
         url = self._base_url / f"{self.prefix}/write/"
         data = {
-            "collection": collection,
+            "schema_name": schema_name,
             "records": docs,
         }
         response = self._restclient.post(url, json=data)
@@ -238,9 +242,9 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
 
     def save(self, records: Records) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
-        collection = records["collection"]
+        schema_name = records["schema_name"]
         records = records["records"]
-        buffer = self._data_buffers[collection]
+        buffer = self._data_buffers[schema_name]
         with self._lock:
             buffer.add_documents(records)
             if buffer.should_flush():
@@ -249,15 +253,17 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
                     len(buffer.data),
                     tags=LogTags.DATALAKE,
                 )
-                self._send_documents(collection, buffer.data)
+                self._send_documents(schema_name, buffer.data)
                 buffer.reset()
         return records
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def _send_documents(self, collection: str, docs: list[dict]) -> list[dict]:
+    def _send_documents(
+        self, schema_name: str, docs: list[dict]
+    ) -> list[dict]:
         url = self._base_url / f"{self.prefix}/write/"
         data = {
-            "collection": collection,
+            "schema_name": schema_name,
             "records": docs,
         }
         response = self._restclient.post(url, json=data)

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/__init__.py

@@ -27,13 +27,6 @@ from splight_lib.models._v4.component import (
     get_field_value,
 )
 from splight_lib.models._v4.data_address import DataAddresses as DataAddress
-from splight_lib.models._v4.datalake import (
-    DataRecords,
-    DataRequest,
-    PipelineStep,
-    Trace,
-    TraceType,
-)
 from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.external_grid import ExternalGrid
 from splight_lib.models._v4.file import File
@@ -97,8 +90,6 @@ __all__ = [
     "SolutionOutputDocument",
     "DataAddress",
     "InputDataAddress",
-    "PipelineStep",
-    "Trace",
     "File",
     "HubComponent",
     "HubServer",
@@ -128,7 +119,4 @@ __all__ = [
     "SolutionOutput",
     "Solution",
     "SolutionConfiguration",
-    "DataRequest",
-    "DataRecords",
-    "TraceType",
 ]

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/component.py

@@ -152,7 +152,7 @@ class ComponentObject(SplightObject):
 
 
 class RoutineEvaluation(SplightDatalakeBaseModel):
-    _collection_name = "routineEvaluations"
+    _schema_name = "routineEvaluations"
 
     routine: str
     status: RoutineStatus

splight_lib-5.24.0.dev1/splight_lib/models/_v4/datalake_base.py

@@ -0,0 +1,131 @@
+from datetime import datetime, timezone
+from typing import ClassVar, Dict
+
+import pandas as pd
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import Self
+
+from splight_lib.client.datalake.v4.builder import get_datalake_client
+from splight_lib.client.datalake.v4.models import (
+    DataReadRequest,
+    DataWriteRequest,
+    DefaultKeys,
+    SolutionKeys,
+)
+
+
+class SplightDatalakeBaseModel(BaseModel):
+    timestamp: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc)
+    )
+    _schema_name: ClassVar[str] = "DatalakeModel"
+
+    model_config = ConfigDict(extra="ignore")
+
+    @classmethod
+    def _get(
+        cls,
+        key_entries: list[dict[str, str]],
+        **params: dict,
+    ) -> list[Self]:
+        request = cls.__to_read_request(
+            key_entries,
+            **params,
+        )
+        dl_client = get_datalake_client()
+        request = request.model_dump(mode="json")
+        response = dl_client.get(request)
+        return response["results"]
+
+    @classmethod
+    async def _async_get(
+        cls,
+        key_entries: list[dict[str, str]],
+        **params: dict,
+    ) -> list[Self]:
+        request = cls.__to_read_request(
+            key_entries,
+            **params,
+        )
+        dl_client = get_datalake_client()
+        request = request.model_dump(mode="json")
+        response = await dl_client.async_get(request)
+        return response["results"]
+
+    @classmethod
+    def _get_dataframe(
+        cls,
+        key_entries: list[dict[str, str]],
+        **params: dict,
+    ) -> pd.DataFrame:
+        request = cls.__to_read_request(
+            key_entries,
+            **params,
+        )
+        dl_client = get_datalake_client()
+        request = request.model_dump(mode="json")
+        response = dl_client.get(request)
+        df = pd.DataFrame(response["results"])
+        if not df.empty:
+            df.index = df["timestamp"]
+            df.drop(columns="timestamp", inplace=True)
+        return df
+
+    @classmethod
+    def __to_read_request(
+        cls,
+        key_entries: list[dict[str, str]],
+        **params: dict,
+    ) -> DataReadRequest:
+        _schema_name = cls._schema_name
+        schema = DefaultKeys if _schema_name == "default" else SolutionKeys
+        return DataReadRequest(
+            keys=schema.load(entries=key_entries),
+            **params,
+        )
+
+    def save(self) -> None:
+        dl_client = get_datalake_client()
+        record = self.__to_write_request()
+        dl_client.save(record.model_dump(mode="json"))
+
+    async def async_save(self) -> None:
+        dl_client = get_datalake_client()
+        record = self.__to_write_request()
+        await dl_client.async_save(record.model_dump(mode="json"))
+
+    @classmethod
+    def save_dataframe(cls, df: pd.DataFrame):
+        df = _fix_dataframe_timestamp(df)
+        instances = df.to_dict("records")
+        records = DataWriteRequest(
+            schema_name=cls._schema_name,
+            records=instances,
+        )
+        dl_client = get_datalake_client()
+        dl_client.save(records.model_dump(mode="json"))
+
+    def dict(self, *args, **kwargs) -> Dict:
+        d = super().model_dump(*args, **kwargs)
+        return {
+            k: v["id"] if isinstance(v, dict) and "id" in v.keys() else v
+            for k, v in d.items()
+        }
+
+    def __to_write_request(self) -> DataWriteRequest:
+        return DataWriteRequest(
+            schema_name=self._schema_name,
+            records=[self.model_dump(mode="json")],
+        )
+
+
+def _fix_dataframe_timestamp(df: pd.DataFrame) -> pd.DataFrame:
+    if df["timestamp"][0].tz is None:
+        df["timestamp"] = df["timestamp"].apply(
+            lambda x: x.tz_localize(tz="UTC").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        )
+    else:
+        df["timestamp"] = df["timestamp"].apply(
+            lambda x: x.tz_convert(tz="UTC").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        )
+    return df

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/exceptions.py

@@ -72,11 +72,5 @@ class InvalidServerConfigType(Exception):
         super().__init__(msg)
 
 
-class TraceAlreadyExistsError(Exception):
-    def __init__(self, ref_id: str):
-        msg = f"Trace with ref_id {ref_id} already exists"
-        super().__init__(msg)
-
-
 class CommandDisable(Exception):
     pass

{splight_lib-5.23.3 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/native.py

@@ -14,9 +14,8 @@ class NativeOutput(SplightDatalakeBaseModel):
     asset: str | Asset
     attribute: str | Attribute
     output_format: str | None = None
-    _collection_name: ClassVar[Literal["default"]] = "default"
+    _schema_name: ClassVar[Literal["default"]] = "default"
     _output_format: ClassVar[str] = "default"
-    _model_type: ClassVar[str] = "attribute_document"
 
     @field_validator("output_format", mode="before")
     def set_output_format(cls, v) -> str:
@@ -26,14 +25,16 @@ class NativeOutput(SplightDatalakeBaseModel):
     def get(
         cls, asset: str | Asset, attribute: str | Attribute, **params: dict
     ) -> list[Self]:
-        return super()._get({"asset": asset, "attribute": attribute}, **params)
+        return super()._get(
+            [{"asset": asset, "attribute": attribute}], **params
+        )
 
     @classmethod
     async def async_get(
         cls, asset: str | Asset, attribute: str | Attribute, **params: dict
     ) -> list[Self]:
         return await super()._async_get(
-            {"asset": asset, "attribute": attribute}, **params
+            [{"asset": asset, "attribute": attribute}], **params
         )
 
     @classmethod
@@ -41,7 +42,7 @@ class NativeOutput(SplightDatalakeBaseModel):
         cls, asset: str | Asset, attribute: str | Attribute, **params: dict
     ) -> pd.DataFrame:
         df = super()._get_dataframe(
-            {"asset": asset, "attribute": attribute}, **params
+            [{"asset": asset, "attribute": attribute}], **params
         )
         df["output_format"] = cls._output_format
         return df
@@ -59,7 +60,7 @@ class NativeOutput(SplightDatalakeBaseModel):
         result = cls.get(
             asset=asset,
             attribute=attribute,
-            from_timestamp=from_timestamp,
+            start=from_timestamp,
             limit=1,
         )
         if result:
@@ -91,8 +92,7 @@ class SolutionOutputDocument(SplightDatalakeBaseModel):
     output: str
     value: bool | int | float | str
 
-    _collection_name: ClassVar[Literal["solutions"]] = "solutions"
-    _model_type: ClassVar[str] = "solution_output_document"
+    _schema_name: ClassVar[Literal["solutions"]] = "solutions"
 
     @classmethod
     def get(

splight_lib-5.23.3/splight_lib/client/datalake/v4/exceptions.py

@@ -1,12 +0,0 @@
-class DatalakeRequestError(Exception):
-    def __init__(self, status_code: int, message: str):
-        self._msg = f"Request failed with status code {status_code}: {message}"
-        super().__init__(self._msg)
-
-
-class InvalidCollectionName(Exception):
-    def __init__(self, collection: str):
-        self._msg = f"Collection {collection} is not a valid collection"
-
-    def __str__(self) -> str:
-        return self._msg