splight-lib 5.24.0__tar.gz → 5.24.0.dev1__tar.gz

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splight-lib
-Version: 5.24.0
+Version: 5.24.0.dev1
 Summary: Splight Python Library
 Author-email: Splight Dev <dev@splight.com>
 License-File: LICENSE.txt

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splight-lib"
-version = "5.24.0"
+version = "5.24.0.dev1"
 description = "Splight Python Library"
 authors = [
     {name = "Splight Dev",email = "dev@splight.com"}

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/common/abstract.py

@@ -1,8 +1,14 @@
 from abc import abstractmethod
+from typing import Any, TypedDict
 
 from splight_lib.abstract.client import AbstractRemoteClient, QuerySet
 
 
+class Records(TypedDict):
+    schema_name: str
+    records: list[dict[str, Any]]
+
+
 # TODO: Fix this class after delete QuerySet
 class AbstractDatalakeClient(AbstractRemoteClient):
     def get(self, *args, **kwargs) -> QuerySet:
@@ -16,11 +22,11 @@ class AbstractDatalakeClient(AbstractRemoteClient):
         return await self._async_get(*args, **kwargs)
 
     @abstractmethod
-    def save(self, records: dict) -> list[dict]:
+    def save(self, records: Records) -> list[dict]:
         pass
 
     @abstractmethod
-    async def async_save(self, records: dict) -> list[dict]:
+    async def async_save(self, records: Records) -> list[dict]:
         pass
 
     @abstractmethod

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/v3/remote_client.py

@@ -2,10 +2,14 @@ from threading import Lock, Thread
 from time import sleep
 
 from furl import furl
+from httpx import HTTPTransport
 from retry import retry
 
 from splight_lib.auth import SplightAuthToken
-from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.common.abstract import (
+    AbstractDatalakeClient,
+    Records,
+)
 from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
 from splight_lib.client.datalake.v3.classmap import COLLECTION_PREFIXS_MAP
 from splight_lib.client.datalake.v3.exceptions import DatalakeRequestError
@@ -38,14 +42,16 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         self._api_version = "v3"
         self._default_path = "data"
 
-        self._restclient = SplightRestClient()
+        self._restclient = SplightRestClient(
+            transport=HTTPTransport(retries=3)
+        )
         self._restclient.update_headers(token.header)
         logger.debug(
             "Remote datalake client initialized.", tags=LogTags.DATALAKE
         )
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def save(self, records: dict) -> list[dict]:
+    def save(self, records: Records) -> list[dict]:
         prefix = self._get_prefix(records["collection"])
         url = self._base_url / f"{prefix}/write/"
         response = self._restclient.post(url, json=records)
@@ -56,7 +62,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
     async def async_save(
         self,
-        records: dict,
+        records: Records,
     ) -> list[dict]:
         # POST /data/write
         prefix = self._get_prefix(records["collection"])
@@ -138,7 +144,7 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
             tags=LogTags.DATALAKE,
         )
 
-    def save(self, records: dict) -> list[dict]:
+    def save(self, records: Records) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
         collection = records["collection"]
         instances = records["records"]
@@ -237,7 +243,7 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
             tags=LogTags.DATALAKE,
         )
 
-    def save(self, records: dict) -> list[dict]:
+    def save(self, records: Records) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
         collection = records["collection"]
         buffer = self._data_buffers[collection]

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/v4/generic.py

@@ -3,9 +3,7 @@ from enum import Enum
 from typing import TypeAlias
 
 Timestamp: TypeAlias = datetime
-# Order in types is important. When parsing if float is first,
-# bools will be interpreted as floats (1.0, 0.0) instead of bools.
-Value: TypeAlias = bool | str | float
+Value: TypeAlias = float | bool | str | int
 
 
 class AggregationFunction(str, Enum):
@@ -14,7 +12,6 @@ class AggregationFunction(str, Enum):
     MIN = "min"
     MAX = "max"
     COUNT = "count"
-    LAST = "last"
 
 
 class TimeUnit(str, Enum):

splight_lib-5.24.0.dev1/splight_lib/client/datalake/v4/models.py

@@ -0,0 +1,78 @@
+from typing import Annotated, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from splight_lib.client.datalake.v4.generic import (
+    AggregationFunction,
+    Timestamp,
+    TimeUnit,
+    TransitionSchemaName,
+    TransitionSort,
+    Value,
+)
+
+
+class DefaultEntryKey(BaseModel):
+    asset: str
+    attribute: str
+
+
+class DefaultKeys(BaseModel):
+    schema_name: Literal[TransitionSchemaName.DEFAULT]
+    entries: list[DefaultEntryKey]
+
+    @classmethod
+    def load(cls, entries: list) -> "DefaultKeys":
+        return cls(
+            schema_name=TransitionSchemaName.DEFAULT,
+            entries=entries,
+        )
+
+
+class SolutionEntryKey(BaseModel):
+    solution: str
+    asset: str
+    output: str
+
+
+class SolutionKeys(BaseModel):
+    schema_name: Literal[TransitionSchemaName.SOLUTIONS]
+    entries: list[SolutionEntryKey]
+
+    @classmethod
+    def load(cls, entries: list) -> "SolutionKeys":
+        return cls(
+            schema_name=TransitionSchemaName.SOLUTIONS,
+            entries=entries,
+        )
+
+
+QueryKeys = Annotated[
+    DefaultKeys | SolutionKeys, Field(discriminator="schema_name")
+]
+
+
+class DataReadRequest(BaseModel):
+    keys: QueryKeys
+    start: Optional[Timestamp]
+    end: Optional[Timestamp] = None
+    time_window_unit: TimeUnit = TimeUnit.SECOND
+    time_window_size: int = 1
+    aggregation: AggregationFunction = AggregationFunction.MAX
+    sort: TransitionSort = TransitionSort.DESC
+    limit: int = 1000
+
+
+class DefaultRecord(DefaultEntryKey):
+    value: Value
+    timestamp: Timestamp
+
+
+class SolutionRecord(SolutionEntryKey):
+    value: Value
+    timestamp: Timestamp
+
+
+class DataWriteRequest(BaseModel):
+    schema_name: TransitionSchemaName
+    records: list[SolutionRecord] | list[DefaultRecord]

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/client/datalake/v4/remote_client.py

@@ -2,12 +2,19 @@ from threading import Lock, Thread
 from time import sleep
 
 from furl import furl
+from httpx import HTTPTransport
 from retry import retry
 
 from splight_lib.auth import SplightAuthToken
-from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.common.abstract import (
+    AbstractDatalakeClient,
+    Records,
+)
 from splight_lib.client.datalake.common.buffer import DatalakeDocumentBuffer
 from splight_lib.client.datalake.v4.exceptions import DatalakeRequestError
+from splight_lib.client.datalake.v4.models import (
+    DataReadRequest,
+)
 from splight_lib.client.exceptions import SPLIGHT_REQUEST_EXCEPTIONS
 from splight_lib.logging._internal import LogTags, get_splight_logger
 from splight_lib.restclient import SplightRestClient
@@ -35,7 +42,9 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
             secret_key=secret_key,
         )
 
-        self._restclient = SplightRestClient()
+        self._restclient = SplightRestClient(
+            transport=HTTPTransport(retries=3)
+        )
         self._restclient.update_headers(token.header)
         logger.debug(
             "Remote datalake client initialized.", tags=LogTags.DATALAKE
@@ -61,7 +70,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         return records["records"]
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    def _get(self, request: dict) -> list[dict]:
+    def _get(self, request: DataReadRequest) -> list[dict]:
         url = self._base_url / f"{self.prefix}/read/"
         response = self._restclient.post(url, json=request)
         if response.is_error:
@@ -69,7 +78,7 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
         return response.json()
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
-    async def _async_get(self, request: dict) -> list[dict]:
+    async def _async_get(self, request: DataReadRequest) -> list[dict]:
         url = self._base_url / f"{self.prefix}/read/"
         response = await self._restclient.async_post(url, json=request)
         if response.is_error:
@@ -133,9 +142,8 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
 
     def save(self, records: dict) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
-        instance = records["records"]
-        data_points = instance["data_points"]
-        schema_name = instance["schema_name"]
+        instances = records["records"]
+        schema_name = records["schema_name"]
         buffer = self._data_buffers[schema_name]
         with self._lock:
             if buffer.should_flush():
@@ -145,8 +153,8 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
                 )
                 self._send_documents(schema_name, buffer.data)
                 buffer.reset()
-            buffer.add_documents(data_points)
-        return data_points
+            buffer.add_documents(instances)
+        return instances
 
     def _flusher(self):
         while True:
@@ -171,19 +179,17 @@ class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
     def _send_documents(
-        self, schema_name: str, data_points: list[dict]
+        self, schema_name: str, docs: list[dict]
     ) -> list[dict]:
         url = self._base_url / f"{self.prefix}/write/"
         data = {
-            "records": {
-                "schema_name": schema_name,
-                "data_points": data_points,
-            },
+            "schema_name": schema_name,
+            "records": docs,
         }
         response = self._restclient.post(url, json=data)
         if response.is_error:
             raise DatalakeRequestError(response.status_code, response.text)
-        return data_points
+        return docs
 
 
 class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
@@ -234,14 +240,13 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
             tags=LogTags.DATALAKE,
         )
 
-    def save(self, records: dict) -> list[dict]:
+    def save(self, records: Records) -> list[dict]:
         logger.debug("Saving documents in datalake", tags=LogTags.DATALAKE)
-        instance = records["records"]
-        schema_name = instance["schema_name"]
-        data_points = instance["data_points"]
+        schema_name = records["schema_name"]
+        records = records["records"]
         buffer = self._data_buffers[schema_name]
         with self._lock:
-            buffer.add_documents(data_points)
+            buffer.add_documents(records)
             if buffer.should_flush():
                 logger.debug(
                     "Flushing datalake buffer with %s elements",
@@ -250,20 +255,18 @@ class BufferedSyncRemoteDataClient(SyncRemoteDatalakeClient):
                 )
                 self._send_documents(schema_name, buffer.data)
                 buffer.reset()
-        return data_points
+        return records
 
     @retry(EXCEPTIONS, tries=3, delay=2, jitter=1)
     def _send_documents(
-        self, schema_name: str, data_points: list[dict]
+        self, schema_name: str, docs: list[dict]
     ) -> list[dict]:
         url = self._base_url / f"{self.prefix}/write/"
         data = {
-            "records": {
-                "schema_name": schema_name,
-                "data_points": data_points,
-            }
+            "schema_name": schema_name,
+            "records": docs,
         }
         response = self._restclient.post(url, json=data)
         if response.is_error:
             raise DatalakeRequestError(response.status_code, response.text)
-        return data_points
+        return docs

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/models/__init__.py

@@ -179,12 +179,8 @@ elif api_version == SplightAPIVersion.V4:
         ComponentType,
         CustomType,
         DataAddress,
-        DataReadRequest,
-        DataWriteRequest,
-        DefaultDataPoint,
-        DefaultEntryKey,
-        DefaultKeys,
-        DefaultRecords,
+        DataRecords,
+        DataRequest,
         Endpoint,
         ExternalGrid,
         File,
@@ -200,9 +196,11 @@ elif api_version == SplightAPIVersion.V4:
         Number,
         Output,
         Parameter,
+        PipelineStep,
         PrivacyPolicy,
         ResourceSummary,
         Routine,
+        RoutineEvaluation,
         RoutineObject,
         RoutineObjectInstance,
         Secret,
@@ -211,15 +209,13 @@ elif api_version == SplightAPIVersion.V4:
         SlackLine,
         Solution,
         SolutionConfiguration,
-        SolutionDataPoint,
-        SolutionEntryKey,
-        SolutionKeys,
         SolutionOutput,
         SolutionOutputDocument,
-        SolutionRecords,
         SplightDatalakeBaseModel,
         String,
         Tag,
+        Trace,
+        TraceType,
         Transformer,
         ValueType,
         get_field_value,
@@ -227,20 +223,11 @@ elif api_version == SplightAPIVersion.V4:
 
     __all__ = [
         "Asset",
-        "DataReadRequest",
-        "DataWriteRequest",
-        "DefaultEntryKey",
-        "DefaultKeys",
-        "DefaultRecords",
-        "DefaultDataPoint",
-        "SolutionEntryKey",
-        "SolutionKeys",
-        "SolutionRecords",
-        "SolutionDataPoint",
         "AssetRelationship",
         "Component",
         "RoutineObjectInstance",
         "RoutineObject",
+        "RoutineEvaluation",
         "ComponentObject",
         "ComponentObjectInstance",
         "ComponentType",
@@ -289,6 +276,12 @@ elif api_version == SplightAPIVersion.V4:
         "Solution",
         "SolutionOutput",
         "SolutionConfiguration",
+        "DataRecords",
+        "DataRequest",
+        "PipelineStep",
+        "Trace",
+        "TraceType",
+        "SolutionOutputDocument",
     ]
 else:
     raise ImportError(f"No models available for API version: '{api_version}'.")

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/__init__.py

@@ -21,23 +21,12 @@ from splight_lib.models._v4.component import (
     Parameter,
     PrivacyPolicy,
     Routine,
+    RoutineEvaluation,
     RoutineObject,
     RoutineObjectInstance,
     get_field_value,
 )
 from splight_lib.models._v4.data_address import DataAddresses as DataAddress
-from splight_lib.models._v4.datalake import (
-    DataReadRequest,
-    DataWriteRequest,
-    DefaultDataPoint,
-    DefaultEntryKey,
-    DefaultKeys,
-    DefaultRecords,
-    SolutionDataPoint,
-    SolutionEntryKey,
-    SolutionKeys,
-    SolutionRecords,
-)
 from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.external_grid import ExternalGrid
 from splight_lib.models._v4.file import File
@@ -81,6 +70,7 @@ __all__ = [
     "Component",
     "RoutineObjectInstance",
     "RoutineObject",
+    "RoutineEvaluation",
     "ComponentObject",
     "ComponentObjectInstance",
     "ComponentType",
@@ -129,14 +119,4 @@ __all__ = [
     "SolutionOutput",
     "Solution",
     "SolutionConfiguration",
-    "DefaultEntryKey",
-    "DefaultKeys",
-    "DefaultRecords",
-    "DefaultDataPoint",
-    "SolutionEntryKey",
-    "SolutionKeys",
-    "SolutionRecords",
-    "SolutionDataPoint",
-    "DataReadRequest",
-    "DataWriteRequest",
 ]

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/component.py

@@ -16,6 +16,7 @@ from strenum import LowercaseStrEnum, PascalCaseStrEnum
 from splight_lib.models._v4.asset import Asset
 from splight_lib.models._v4.attribute import Attribute
 from splight_lib.models._v4.data_address import DataAddresses as DLDataAddress
+from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.exceptions import InvalidObjectInstance
 from splight_lib.models._v4.file import File
 from splight_lib.models._v4.secret import Secret
@@ -150,6 +151,14 @@ class ComponentObject(SplightObject):
     data: list[InputParameter] = []
 
 
+class RoutineEvaluation(SplightDatalakeBaseModel):
+    _schema_name = "routineEvaluations"
+
+    routine: str
+    status: RoutineStatus
+    status_text: str | None
+
+
 class RoutineObject(SplightObject):
     status: RoutineStatus = RoutineStatus.RUNNING
 
@@ -157,6 +166,29 @@ class RoutineObject(SplightObject):
     input: list[InputDataAddress] = []
     output: list[InputDataAddress] = []
 
+    def report_status(
+        self, status: RoutineStatus, status_text: str | None = None
+    ):
+        evaluation_status = RoutineEvaluation(
+            routine=str(self.id),
+            status=status,
+            status_text=status_text,
+        )
+        evaluation_status.save()
+
+        if self.status != status:
+            self.status = status
+            self._update_status()
+
+    def _update_status(self):
+        _ = self._db_client.operate(
+            resource_name="routine-status",
+            instance={
+                "routine": self.id,
+                "status": self.status,
+            },
+        )
+
 
 class Component(SplightDatabaseBaseModel):
     id: str | None = None

{splight_lib-5.24.0 → splight_lib-5.24.0.dev1}/splight_lib/models/_v4/datalake_base.py

@@ -5,13 +5,12 @@ import pandas as pd
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import Self
 
-from splight_lib.models._v4.datalake import (
+from splight_lib.client.datalake.v4.builder import get_datalake_client
+from splight_lib.client.datalake.v4.models import (
     DataReadRequest,
     DataWriteRequest,
     DefaultKeys,
-    DefaultRecords,
     SolutionKeys,
-    SolutionRecords,
 )
 
 
@@ -33,7 +32,10 @@ class SplightDatalakeBaseModel(BaseModel):
             key_entries,
             **params,
         )
-        return request.apply()
+        dl_client = get_datalake_client()
+        request = request.model_dump(mode="json")
+        response = dl_client.get(request)
+        return response["results"]
 
     @classmethod
     async def _async_get(
@@ -45,7 +47,10 @@ class SplightDatalakeBaseModel(BaseModel):
             key_entries,
             **params,
         )
-        return await request.async_apply()
+        dl_client = get_datalake_client()
+        request = request.model_dump(mode="json")
+        response = await dl_client.async_get(request)
+        return response["results"]
 
     @classmethod
     def _get_dataframe(
@@ -57,8 +62,10 @@ class SplightDatalakeBaseModel(BaseModel):
             key_entries,
             **params,
         )
-        results = request.apply()
-        df = pd.DataFrame(results)
+        dl_client = get_datalake_client()
+        request = request.model_dump(mode="json")
+        response = dl_client.get(request)
+        df = pd.DataFrame(response["results"])
         if not df.empty:
             df.index = df["timestamp"]
             df.drop(columns="timestamp", inplace=True)
@@ -77,37 +84,26 @@ class SplightDatalakeBaseModel(BaseModel):
             **params,
         )
 
-    @classmethod
-    def __to_write_request(
-        cls,
-        data_points: list[dict[str, str]],
-    ) -> DataWriteRequest:
-        _schema_name = cls._schema_name
-        schema = (
-            DefaultRecords if _schema_name == "default" else SolutionRecords
-        )
-        return DataWriteRequest(
-            records=schema.load(data_points=data_points),
-        )
-
     def save(self) -> None:
-        request = self.__to_write_request(
-            data_points=[self.model_dump(mode="json")]
-        )
-        request.apply()
+        dl_client = get_datalake_client()
+        record = self.__to_write_request()
+        dl_client.save(record.model_dump(mode="json"))
 
     async def async_save(self) -> None:
-        request = self.__to_write_request(
-            data_points=[self.model_dump(mode="json")]
-        )
-        await request.async_apply()
+        dl_client = get_datalake_client()
+        record = self.__to_write_request()
+        await dl_client.async_save(record.model_dump(mode="json"))
 
     @classmethod
     def save_dataframe(cls, df: pd.DataFrame):
         df = _fix_dataframe_timestamp(df)
         instances = df.to_dict("records")
-        request = cls.__to_write_request(data_points=instances)
-        request.apply()
+        records = DataWriteRequest(
+            schema_name=cls._schema_name,
+            records=instances,
+        )
+        dl_client = get_datalake_client()
+        dl_client.save(records.model_dump(mode="json"))
 
     def dict(self, *args, **kwargs) -> Dict:
         d = super().model_dump(*args, **kwargs)
@@ -116,6 +112,12 @@ class SplightDatalakeBaseModel(BaseModel):
             for k, v in d.items()
         }
 
+    def __to_write_request(self) -> DataWriteRequest:
+        return DataWriteRequest(
+            schema_name=self._schema_name,
+            records=[self.model_dump(mode="json")],
+        )
+
 
 def _fix_dataframe_timestamp(df: pd.DataFrame) -> pd.DataFrame:
     if df["timestamp"][0].tz is None: