splight-lib 5.24.0.dev1__tar.gz → 5.24.0.dev3__tar.gz

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splight-lib
-Version: 5.24.0.dev1
+Version: 5.24.0.dev3
 Summary: Splight Python Library
 Author-email: Splight Dev <dev@splight.com>
 License-File: LICENSE.txt

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splight-lib"
-version = "5.24.0.dev1"
+version = "5.24.0.dev3"
 description = "Splight Python Library"
 authors = [
     {name = "Splight Dev",email = "dev@splight.com"}

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/client/datalake/common/abstract.py

@@ -1,14 +1,8 @@
 from abc import abstractmethod
-from typing import Any, TypedDict
 
 from splight_lib.abstract.client import AbstractRemoteClient, QuerySet
 
 
-class Records(TypedDict):
-    schema_name: str
-    records: list[dict[str, Any]]
-
-
 # TODO: Fix this class after delete QuerySet
 class AbstractDatalakeClient(AbstractRemoteClient):
     def get(self, *args, **kwargs) -> QuerySet:
@@ -22,11 +16,11 @@ class AbstractDatalakeClient(AbstractRemoteClient):
         return await self._async_get(*args, **kwargs)
 
     @abstractmethod
-    def save(self, records: Records) -> list[dict]:
+    def save(self, records: dict) -> list[dict]:
         pass
 
     @abstractmethod
-    async def async_save(self, records: Records) -> list[dict]:
+    async def async_save(self, records: dict) -> list[dict]:
         pass
 
     @abstractmethod

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/client/datalake/v3/remote_client.py

@@ -6,10 +6,7 @@ from httpx import HTTPTransport
 from retry import retry
 
 from splight_lib.auth import SplightAuthToken
-from splight_lib.client.datalake.common.abstract import (
-    AbstractDatalakeClient,
-    Records,
-)
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
 from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
 from splight_lib.client.datalake.v3.classmap import COLLECTION_PREFIXS_MAP
 from splight_lib.client.datalake.v3.exceptions import DatalakeRequestError
@@ -51,7 +48,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         )
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def save(self, records: Records) -> list[dict]:
+    def save(self, records: dict) -> list[dict]:
         prefix = self._get_prefix(records["collection"])
         url = self._base_url / f"{prefix}/write/"
         response = self._restclient.post(url, json=records)
@@ -62,7 +59,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
     async def async_save(
         self,
-        records: Records,
+        records: dict,
     ) -> list[dict]:
         # POST /data/write
         prefix = self._get_prefix(records["collection"])
@@ -144,7 +141,7 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
             tags=LogTags.DATALAKE,
         )
 
-    def save(self, records: Records) -> list[dict]:
+    def save(self, records: dict) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
         collection = records["collection"]
         instances = records["records"]
@@ -243,7 +240,7 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
             tags=LogTags.DATALAKE,
         )
 
-    def save(self, records: Records) -> list[dict]:
+    def save(self, records: dict) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
         collection = records["collection"]
         buffer = self._data_buffers[collection]

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/client/datalake/v4/generic.py

@@ -3,7 +3,9 @@ from enum import Enum
 from typing import TypeAlias
 
 Timestamp: TypeAlias = datetime
-Value: TypeAlias = float | bool | str | int
+# Order in types is important. When parsing if float is first,
+# bools will be interpreted as floats (1.0, 0.0) instead of bools.
+Value: TypeAlias = bool | str | float
 
 
 class AggregationFunction(str, Enum):

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/client/datalake/v4/remote_client.py

@@ -6,15 +6,9 @@ from httpx import HTTPTransport
 from retry import retry
 
 from splight_lib.auth import SplightAuthToken
-from splight_lib.client.datalake.common.abstract import (
-    AbstractDatalakeClient,
-    Records,
-)
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
 from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
 from splight_lib.client.datalake.v4.exceptions import DatalakeRequestError
-from splight_lib.client.datalake.v4.models import (
-    DataReadRequest,
-)
 from splight_lib.client.exceptions import SPLIGHT_REQUEST_EXCEPTIONS
 from splight_lib.logging._internal import LogTags, get_splight_logger
 from splight_lib.restclient import SplightRestClient
@@ -70,7 +64,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         return records["records"]
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def _get(self, request: DataReadRequest) -> list[dict]:
+    def _get(self, request: dict) -> list[dict]:
         url = self._base_url / f"{self.prefix}/read/"
         response = self._restclient.post(url, json=request)
         if response.is_error:
@@ -78,7 +72,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         return response.json()
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    async def _async_get(self, request: DataReadRequest) -> list[dict]:
+    async def _async_get(self, request: dict) -> list[dict]:
         url = self._base_url / f"{self.prefix}/read/"
         response = await self._restclient.async_post(url, json=request)
         if response.is_error:
@@ -142,8 +136,9 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
 
     def save(self, records: dict) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
-        instances = records["records"]
-        schema_name = records["schema_name"]
+        instance = records["records"]
+        data_points = instance["data_points"]
+        schema_name = instance["schema_name"]
         buffer = self._data_buffers[schema_name]
         with self._lock:
             if buffer.should_flush():
@@ -153,8 +148,8 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
                 )
                 self._send_documents(schema_name, buffer.data)
                 buffer.reset()
-            buffer.add_documents(instances)
-        return instances
+            buffer.add_documents(data_points)
+        return data_points
 
     def _flusher(self):
         while True:
@@ -179,17 +174,19 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
     def _send_documents(
-        self, schema_name: str, docs: list[dict]
+        self, schema_name: str, data_points: list[dict]
     ) -> list[dict]:
         url = self._base_url / f"{self.prefix}/write/"
         data = {
-            "schema_name": schema_name,
-            "records": docs,
+            "records": {
+                "schema_name": schema_name,
+                "data_points": data_points,
+            },
         }
         response = self._restclient.post(url, json=data)
         if response.is_error:
             raise DatalakeRequestError(response.status_code, response.text)
-        return docs
+        return data_points
 
 
 class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
@@ -240,13 +237,14 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
             tags=LogTags.DATALAKE,
         )
 
-    def save(self, records: Records) -> list[dict]:
+    def save(self, records: dict) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
-        schema_name = records["schema_name"]
-        records = records["records"]
+        instance = records["records"]
+        schema_name = instance["schema_name"]
+        data_points = instance["data_points"]
         buffer = self._data_buffers[schema_name]
         with self._lock:
-            buffer.add_documents(records)
+            buffer.add_documents(data_points)
             if buffer.should_flush():
                 logger.debug(
                     "Flushing datalake buffer with %s elements",
@@ -255,18 +253,20 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
                 )
                 self._send_documents(schema_name, buffer.data)
                 buffer.reset()
-        return records
+        return data_points
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
     def _send_documents(
-        self, schema_name: str, docs: list[dict]
+        self, schema_name: str, data_points: list[dict]
     ) -> list[dict]:
         url = self._base_url / f"{self.prefix}/write/"
         data = {
-            "schema_name": schema_name,
-            "records": docs,
+            "records": {
+                "schema_name": schema_name,
+                "data_points": data_points,
+            }
         }
         response = self._restclient.post(url, json=data)
         if response.is_error:
             raise DatalakeRequestError(response.status_code, response.text)
-        return docs
+        return data_points

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/models/__init__.py

@@ -179,8 +179,12 @@ elif api_version == SplightAPIVersion.V4:
         ComponentType,
         CustomType,
         DataAddress,
-        DataRecords,
-        DataRequest,
+        DataReadRequest,
+        DataWriteRequest,
+        DefaultDataPoint,
+        DefaultEntryKey,
+        DefaultKeys,
+        DefaultRecords,
         Endpoint,
         ExternalGrid,
         File,
@@ -196,11 +200,9 @@ elif api_version == SplightAPIVersion.V4:
         Number,
         Output,
         Parameter,
-        PipelineStep,
         PrivacyPolicy,
         ResourceSummary,
         Routine,
-        RoutineEvaluation,
         RoutineObject,
         RoutineObjectInstance,
         Secret,
@@ -209,13 +211,15 @@ elif api_version == SplightAPIVersion.V4:
         SlackLine,
         Solution,
         SolutionConfiguration,
+        SolutionDataPoint,
+        SolutionEntryKey,
+        SolutionKeys,
         SolutionOutput,
         SolutionOutputDocument,
+        SolutionRecords,
         SplightDatalakeBaseModel,
         String,
         Tag,
-        Trace,
-        TraceType,
         Transformer,
         ValueType,
         get_field_value,
@@ -223,11 +227,20 @@ elif api_version == SplightAPIVersion.V4:
 
     __all__ = [
         "Asset",
+        "DataReadRequest",
+        "DataWriteRequest",
+        "DefaultEntryKey",
+        "DefaultKeys",
+        "DefaultRecords",
+        "DefaultDataPoint",
+        "SolutionEntryKey",
+        "SolutionKeys",
+        "SolutionRecords",
+        "SolutionDataPoint",
         "AssetRelationship",
         "Component",
         "RoutineObjectInstance",
         "RoutineObject",
-        "RoutineEvaluation",
         "ComponentObject",
         "ComponentObjectInstance",
         "ComponentType",
@@ -276,11 +289,6 @@ elif api_version == SplightAPIVersion.V4:
         "Solution",
         "SolutionOutput",
         "SolutionConfiguration",
-        "DataRecords",
-        "DataRequest",
-        "PipelineStep",
-        "Trace",
-        "TraceType",
         "SolutionOutputDocument",
     ]
 else:

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/models/_v4/__init__.py

@@ -21,12 +21,23 @@ from splight_lib.models._v4.component import (
     Parameter,
     PrivacyPolicy,
     Routine,
-    RoutineEvaluation,
     RoutineObject,
     RoutineObjectInstance,
     get_field_value,
 )
 from splight_lib.models._v4.data_address import DataAddresses as DataAddress
+from splight_lib.models._v4.datalake import (
+    DataReadRequest,
+    DataWriteRequest,
+    DefaultDataPoint,
+    DefaultEntryKey,
+    DefaultKeys,
+    DefaultRecords,
+    SolutionDataPoint,
+    SolutionEntryKey,
+    SolutionKeys,
+    SolutionRecords,
+)
 from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.external_grid import ExternalGrid
 from splight_lib.models._v4.file import File
@@ -70,7 +81,6 @@ __all__ = [
     "Component",
     "RoutineObjectInstance",
     "RoutineObject",
-    "RoutineEvaluation",
     "ComponentObject",
     "ComponentObjectInstance",
     "ComponentType",
@@ -119,4 +129,14 @@ __all__ = [
     "SolutionOutput",
     "Solution",
     "SolutionConfiguration",
+    "DefaultEntryKey",
+    "DefaultKeys",
+    "DefaultRecords",
+    "DefaultDataPoint",
+    "SolutionEntryKey",
+    "SolutionKeys",
+    "SolutionRecords",
+    "SolutionDataPoint",
+    "DataReadRequest",
+    "DataWriteRequest",
 ]

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/models/_v4/component.py

@@ -16,7 +16,6 @@ from strenum import LowercaseStrEnum, PascalCaseStrEnum
 from splight_lib.models._v4.asset import Asset
 from splight_lib.models._v4.attribute import Attribute
 from splight_lib.models._v4.data_address import DataAddresses as DLDataAddress
-from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.exceptions import InvalidObjectInstance
 from splight_lib.models._v4.file import File
 from splight_lib.models._v4.secret import Secret
@@ -151,14 +150,6 @@ class ComponentObject(SplightObject):
     data: list[InputParameter] = []
 
 
-class RoutineEvaluation(SplightDatalakeBaseModel):
-    _schema_name = "routineEvaluations"
-
-    routine: str
-    status: RoutineStatus
-    status_text: str | None
-
-
 class RoutineObject(SplightObject):
     status: RoutineStatus = RoutineStatus.RUNNING
 
@@ -166,29 +157,6 @@ class RoutineObject(SplightObject):
     input: list[InputDataAddress] = []
     output: list[InputDataAddress] = []
 
-    def report_status(
-        self, status: RoutineStatus, status_text: str | None = None
-    ):
-        evaluation_status = RoutineEvaluation(
-            routine=str(self.id),
-            status=status,
-            status_text=status_text,
-        )
-        evaluation_status.save()
-
-        if self.status != status:
-            self.status = status
-            self._update_status()
-
-    def _update_status(self):
-        _ = self._db_client.operate(
-            resource_name="routine-status",
-            instance={
-                "routine": self.id,
-                "status": self.status,
-            },
-        )
-
 
 class Component(SplightDatabaseBaseModel):
     id: str | None = None

splight_lib-5.24.0.dev3/splight_lib/models/_v4/datalake.py

@@ -0,0 +1,133 @@
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, Field
+
+from splight_lib.client.datalake.v4.builder import get_datalake_client
+from splight_lib.client.datalake.v4.generic import (
+    AggregationFunction,
+    Timestamp,
+    TimeUnit,
+    TransitionSchemaName,
+    TransitionSort,
+    Value,
+)
+
+
+class DefaultEntryKey(BaseModel):
+    asset: str
+    attribute: str
+
+
+class DefaultKeys(BaseModel):
+    schema_name: Literal[TransitionSchemaName.DEFAULT]
+    entries: list[DefaultEntryKey]
+
+    @classmethod
+    def load(cls, entries: list) -> "DefaultKeys":
+        return cls(
+            schema_name=TransitionSchemaName.DEFAULT,
+            entries=entries,
+        )
+
+
+class SolutionEntryKey(BaseModel):
+    solution: str
+    asset: str
+    output: str
+
+
+class SolutionKeys(BaseModel):
+    schema_name: Literal[TransitionSchemaName.SOLUTIONS]
+    entries: list[SolutionEntryKey]
+
+    @classmethod
+    def load(cls, entries: list) -> "SolutionKeys":
+        return cls(
+            schema_name=TransitionSchemaName.SOLUTIONS,
+            entries=entries,
+        )
+
+
+QueryKeys = Annotated[
+    DefaultKeys | SolutionKeys, Field(discriminator="schema_name")
+]
+
+
+class DataReadRequest(BaseModel):
+    keys: QueryKeys
+    start: Timestamp | None = None
+    end: Timestamp | None = None
+    time_window_unit: TimeUnit = TimeUnit.SECOND
+    time_window_size: int = 1
+    aggregation: AggregationFunction = AggregationFunction.MAX
+    sort: TransitionSort = TransitionSort.DESC
+    limit: int = 1000
+
+    def apply(self) -> dict:
+        dl_client = get_datalake_client()
+        request = self.model_dump(mode="json")
+        response = dl_client.get(request)
+        return response["results"]
+
+    async def async_apply(self) -> dict:
+        dl_client = get_datalake_client()
+        request = self.model_dump(mode="json")
+        response = await dl_client.async_get(request)
+        return response["results"]
+
+
+class DefaultDataPoint(DefaultEntryKey):
+    value: Value
+    timestamp: Timestamp
+
+
+class DefaultRecords(BaseModel):
+    schema_name: Literal[TransitionSchemaName.DEFAULT] = (
+        TransitionSchemaName.DEFAULT
+    )
+    data_points: list[DefaultDataPoint]
+
+    @classmethod
+    def load(cls, data_points: list[DefaultDataPoint]) -> "DefaultRecords":
+        return cls(
+            schema_name=TransitionSchemaName.DEFAULT,
+            data_points=data_points,
+        )
+
+
+class SolutionDataPoint(SolutionEntryKey):
+    value: Value
+    timestamp: Timestamp
+
+
+class SolutionRecords(BaseModel):
+    schema_name: Literal[TransitionSchemaName.SOLUTIONS] = (
+        TransitionSchemaName.SOLUTIONS
+    )
+    data_points: list[SolutionDataPoint]
+
+    @classmethod
+    def load(cls, data_points: list[SolutionDataPoint]) -> "SolutionRecords":
+        return cls(
+            schema_name=TransitionSchemaName.SOLUTIONS,
+            data_points=data_points,
+        )
+
+
+WriteRecords = Annotated[
+    DefaultRecords | SolutionRecords, Field(discriminator="schema_name")
+]
+
+
+class DataWriteRequest(BaseModel):
+    records: WriteRecords
+
+    def apply(self) -> None:
+        dl_client = get_datalake_client()
+        request = self.model_dump(mode="json")
+        dl_client.save(request)
+
+    async def async_apply(self) -> None:
+        dl_client = get_datalake_client()
+        request = self.model_dump(mode="json")
+        await dl_client.async_save(request)

{splight_lib-5.24.0.dev1 → splight_lib-5.24.0.dev3}/splight_lib/models/_v4/datalake_base.py

@@ -5,12 +5,13 @@ import pandas as pd
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import Self
 
-from splight_lib.client.datalake.v4.builder import get_datalake_client
-from splight_lib.client.datalake.v4.models import (
+from splight_lib.models._v4.datalake import (
     DataReadRequest,
     DataWriteRequest,
     DefaultKeys,
+    DefaultRecords,
     SolutionKeys,
+    SolutionRecords,
 )
 
 
@@ -32,10 +33,7 @@ class SplightDatalakeBaseModel(BaseModel):
             key_entries,
             **params,
         )
-        dl_client = get_datalake_client()
-        request = request.model_dump(mode="json")
-        response = dl_client.get(request)
-        return response["results"]
+        return request.apply()
 
     @classmethod
     async def _async_get(
@@ -47,10 +45,7 @@ class SplightDatalakeBaseModel(BaseModel):
             key_entries,
             **params,
         )
-        dl_client = get_datalake_client()
-        request = request.model_dump(mode="json")
-        response = await dl_client.async_get(request)
-        return response["results"]
+        return await request.async_apply()
 
     @classmethod
     def _get_dataframe(
@@ -62,10 +57,8 @@ class SplightDatalakeBaseModel(BaseModel):
             key_entries,
             **params,
         )
-        dl_client = get_datalake_client()
-        request = request.model_dump(mode="json")
-        response = dl_client.get(request)
-        df = pd.DataFrame(response["results"])
+        results = request.apply()
+        df = pd.DataFrame(results)
         if not df.empty:
             df.index = df["timestamp"]
             df.drop(columns="timestamp", inplace=True)
@@ -84,26 +77,37 @@ class SplightDatalakeBaseModel(BaseModel):
             **params,
         )
 
+    @classmethod
+    def __to_write_request(
+        cls,
+        data_points: list[dict[str, str]],
+    ) -> DataWriteRequest:
+        _schema_name = cls._schema_name
+        schema = (
+            DefaultRecords if _schema_name == "default" else SolutionRecords
+        )
+        return DataWriteRequest(
+            records=schema.load(data_points=data_points),
+        )
+
     def save(self) -> None:
-        dl_client = get_datalake_client()
-        record = self.__to_write_request()
-        dl_client.save(record.model_dump(mode="json"))
+        request = self.__to_write_request(
+            data_points=[self.model_dump(mode="json")]
+        )
+        request.apply()
 
     async def async_save(self) -> None:
-        dl_client = get_datalake_client()
-        record = self.__to_write_request()
-        await dl_client.async_save(record.model_dump(mode="json"))
+        request = self.__to_write_request(
+            data_points=[self.model_dump(mode="json")]
+        )
+        await request.async_apply()
 
     @classmethod
     def save_dataframe(cls, df: pd.DataFrame):
         df = _fix_dataframe_timestamp(df)
         instances = df.to_dict("records")
-        records = DataWriteRequest(
-            schema_name=cls._schema_name,
-            records=instances,
-        )
-        dl_client = get_datalake_client()
-        dl_client.save(records.model_dump(mode="json"))
+        request = cls.__to_write_request(data_points=instances)
+        request.apply()
 
     def dict(self, *args, **kwargs) -> Dict:
         d = super().model_dump(*args, **kwargs)
@@ -112,12 +116,6 @@ class SplightDatalakeBaseModel(BaseModel):
             for k, v in d.items()
         }
 
-    def __to_write_request(self) -> DataWriteRequest:
-        return DataWriteRequest(
-            schema_name=self._schema_name,
-            records=[self.model_dump(mode="json")],
-        )
-
 
 def _fix_dataframe_timestamp(df: pd.DataFrame) -> pd.DataFrame:
     if df["timestamp"][0].tz is None: