splight-lib 5.22.5.dev100__tar.gz → 5.23.0.dev0__tar.gz

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splight-lib
-Version: 5.22.5.dev100
+Version: 5.23.0.dev0
 Summary: Splight Python Library
 Author-email: Splight Dev <dev@splight.com>
 License-File: LICENSE.txt

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splight-lib"
-version = "5.22.5.dev100"
+version = "5.23.0dev0"
 description = "Splight Python Library"
 authors = [
     {name = "Splight Dev",email = "dev@splight.com"}

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/splight_lib/client/datalake/v4/remote_client.py

@@ -26,7 +26,6 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
     def __init__(
         self,
         base_url: str,
-        resource: str,
         access_id: str,
         secret_key: str,
         api_version: SplightAPIVersion = SplightAPIVersion.V4,
@@ -35,7 +34,6 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
     ):
         super().__init__()
         self._base_url = furl(base_url)
-        self.resource = resource
         token = SplightAuthToken(
             access_key=access_id,
             secret_key=secret_key,
@@ -86,7 +84,8 @@ class SyncRemoteDatalakeClient(AbstractDatalakeClient):
 
     @property
     def prefix(self) -> str:
-        return f"v4/data/{self.resource}"
+        return "v3/data/"
+        # return f"v4/data/{self.resource}"
 
 
 class BufferedAsyncRemoteDatalakeClient(SyncRemoteDatalakeClient):

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/splight_lib/models/__init__.py

@@ -169,7 +169,7 @@ elif api_version == SplightAPIVersion.V4:
         AssetKind,
         AssetRelationship,
         Attribute,
-        AttributeDocument,
+        # AttributeDocument,
         AttributeType,
         Battery,
         Boolean,
@@ -196,8 +196,8 @@ elif api_version == SplightAPIVersion.V4:
         Output,
         Parameter,
         PrivacyPolicy,
-        Query,
-        Records,
+        # Query,
+        # Records,
         ResourceSummary,
         Routine,
         RoutineEvaluation,

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/splight_lib/models/_v4/__init__.py

@@ -28,10 +28,15 @@ from splight_lib.models._v4.component import (
 )
 from splight_lib.models._v4.data_address import DataAddresses as DataAddress
 from splight_lib.models._v4.datalake import (
-    AttributeDocument,
-    Query,
-    Records,
-    SolutionOutputDocument,
+    DataRecords,
+    DataRequest,
+    PipelineStep,
+    Trace,
+    TraceType,
+    # AttributeDocument,
+    # Query,
+    # Records,
+    # SolutionOutputDocument,
 )
 from splight_lib.models._v4.datalake_base import SplightDatalakeBaseModel
 from splight_lib.models._v4.external_grid import ExternalGrid
@@ -43,7 +48,12 @@ from splight_lib.models._v4.hub_server import HubServer
 from splight_lib.models._v4.line import Line
 from splight_lib.models._v4.load import Load
 from splight_lib.models._v4.metadata import Metadata
-from splight_lib.models._v4.native import Boolean, Number, String
+from splight_lib.models._v4.native import (
+    Boolean,
+    Number,
+    SolutionOutputDocument,
+    String,
+)
 from splight_lib.models._v4.secret import Secret
 from splight_lib.models._v4.segment import Segment
 from splight_lib.models._v4.server import Server
@@ -113,4 +123,8 @@ __all__ = [
     "Segment",
     "SlackLine",
     "Transformer",
+    "SolutionOutputDocument",
+    "DataRequest",
+    "DataRecords",
+    "TraceType",
 ]

splight_lib-5.23.0.dev0/splight_lib/models/_v4/datalake.py

@@ -0,0 +1,209 @@
+from datetime import datetime
+from enum import Enum
+from hashlib import sha256
+from typing import Annotated, Any, Generator, Generic, Literal, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+from typing_extensions import Self
+
+from splight_lib.client.datalake import DatalakeClientBuilder
+from splight_lib.client.datalake.common.abstract import AbstractDatalakeClient
+from splight_lib.client.datalake.v3.constants import StepName
+from splight_lib.models._v3.asset import Asset
+from splight_lib.models._v3.attribute import Attribute
+from splight_lib.models._v3.exceptions import TraceAlreadyExistsError
+from splight_lib.settings import (
+    SplightAPIVersion,
+    datalake_settings,
+    workspace_settings,
+)
+
+MAX_NUM_TRACES = 500
+T = TypeVar("T")
+
+
+def hash(string: str) -> str:
+    return sha256(string.encode("utf-8")).hexdigest()
+
+
+def get_datalake_client() -> AbstractDatalakeClient:
+    return DatalakeClientBuilder.build(
+        version=SplightAPIVersion.V4,
+        dl_client_type=datalake_settings.DL_CLIENT_TYPE,
+        parameters={
+            "base_url": workspace_settings.SPLIGHT_PLATFORM_API_HOST,
+            "access_id": workspace_settings.SPLIGHT_ACCESS_ID,
+            "secret_key": workspace_settings.SPLIGHT_SECRET_KEY,
+            "api_version": SplightAPIVersion.V4,
+            "buffer_size": datalake_settings.DL_BUFFER_SIZE,
+            "buffer_timeout": datalake_settings.DL_BUFFER_TIMEOUT,
+        },
+    )
+
+
+class TraceType(str, Enum):
+    QUERY = "QUERY"
+    EXPRESSION = "EXPRESSION"
+    METADATA = "METADATA"
+
+    @classmethod
+    def choices(cls):
+        return tuple((i.name, i.value) for i in cls)
+
+
+class PipelineStep(BaseModel):
+    name: StepName
+    operation: str | int | dict[str, Any]
+
+    @classmethod
+    def from_dict(cls, step_dict: dict[str, Any]) -> Self:
+        (name, operation), *aux = step_dict.items()
+        return cls(name=name.lstrip("$"), operation=operation)
+
+    def to_step(self) -> dict[str, dict[str, Any]]:
+        return {f"${self.name.value}": self.operation}
+
+
+class Trace(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)
+    ref_id: str
+    type: TraceType = TraceType.QUERY
+    expression: dict | None = None
+    # TODO: Review if it should be list[PipelineStep]
+    pipeline: list[dict] = []
+    address: Annotated[dict, Field(exclude=True)]
+
+    @classmethod
+    def from_address(
+        cls, asset: str | Asset, attribute: str | Attribute
+    ) -> Self:
+        asset_id = asset.id if isinstance(asset, Asset) else asset
+        attribute_id = (
+            attribute.id if isinstance(attribute, Attribute) else attribute
+        )
+        return cls(
+            ref_id=hash(f"{asset_id}_{attribute_id}"),
+            type=TraceType.QUERY,
+            pipeline=[
+                PipelineStep(
+                    name=StepName.MATCH,
+                    operation={"asset": asset_id, "attribute": attribute_id},
+                ).to_step()
+            ],
+            address={"asset": asset_id, "attribute": attribute_id},
+        )
+
+    @classmethod
+    def from_so_filter(cls, asset: str, solution: str, output: str) -> Self:
+        return cls(
+            ref_id=hash(f"{asset}_{solution}_{output}"),
+            type=TraceType.QUERY,
+            pipeline=[
+                PipelineStep(
+                    name=StepName.MATCH,
+                    operation={
+                        "asset": asset,
+                        "solution": solution,
+                        "output": output,
+                    },
+                ).to_step()
+            ],
+            address={"asset": asset, "solution": solution, "output": output},
+        )
+
+    def add_step(self, step: PipelineStep) -> None:
+        self.pipeline.append(step.to_step())
+
+
+class DataRequest(Generic[T], BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    collection: str = "default"
+    sort_field: str = "timestamp"
+    sort_direction: Literal[-1, 1] = -1
+    limit: Annotated[int, Field(ge=1, le=10000)] = 10000
+    max_time_ms: Annotated[int, Field(ge=1, le=10000)] = 10000
+    from_timestamp: datetime | None = None
+    to_timestamp: datetime | None = None
+    traces: list[Trace] = []
+
+    _dl_client: AbstractDatalakeClient = PrivateAttr()
+    _traces_ref: dict[str, dict] = {}
+
+    def add_trace(self, trace: Trace) -> None:
+        if trace.ref_id in self._traces_ref:
+            raise TraceAlreadyExistsError(trace.ref_id)
+        self.traces.append(trace)
+        self._traces_ref.update({trace.ref_id: trace.address})
+
+    def as_pipeline(self) -> list[dict[str, Any]]:
+        return [step.to_step() for step in self.pipeline]
+
+    def apply(self) -> list[T]:
+        dl_client = get_datalake_client()
+        request = self.model_dump(mode="json")
+        traces = request.pop("traces")
+        data = []
+        for batch in chunk_list(traces, MAX_NUM_TRACES):
+            request["traces"] = batch
+            response = dl_client.get(request)
+            data.extend(self._parse_respose(response["results"]))
+        return data
+
+    async def async_apply(self) -> list[T]:
+        dl_client = get_datalake_client()
+        request = self.model_dump(mode="json")
+        traces = request.pop("traces")
+        data = []
+        for batch in chunk_list(traces, MAX_NUM_TRACES):
+            request["traces"] = batch
+            response = await dl_client.async_get(request)
+            data.extend(self._parse_respose(response["results"]))
+        return data
+
+    def _parse_respose(self, response: dict) -> list[T]:
+        model_class = self.__orig_class__.__args__[0]
+        data = []
+        for item in response:
+            timestamp = item.pop("timestamp")
+            values = [
+                model_class(
+                    timestamp=timestamp, value=value, **self._traces_ref[key]
+                )
+                for key, value in item.items()
+                if value is not None
+            ]
+            data.extend(values)
+        return data
+
+
+class DataRecords(BaseModel):
+    collection: str = "default"
+    records: list[dict[str, Any]] = []
+
+    def apply(self) -> None:
+        dl_client = get_datalake_client()
+        dl_client.save(self.model_dump(mode="json"))
+
+    async def async_apply(self) -> None:
+        dl_client = get_datalake_client()
+        await dl_client.async_save(self.model_dump(mode="json"))
+
+
+def chunk_list(
+    datas: list[Any], chunksize: int
+) -> Generator[list[Any], None, None]:
+    """Split list into chunks
+
+    Parameters
+    ----------
+    datas : list[Any]
+        the list of data
+    chunksize : int
+        the size of chunk
+
+    Returns
+    -------
+    Generator with the chunks
+    """
+    for i in range(0, len(datas), chunksize):
+        yield datas[i : i + chunksize]

splight_lib-5.23.0.dev0/splight_lib/models/_v4/datalake_base.py

@@ -0,0 +1,141 @@
+from datetime import datetime, timezone
+from typing import Any, ClassVar, Dict, TypeVar
+
+import pandas as pd
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import Self
+
+from splight_lib.models._v4.datalake import (
+    DataRecords,
+    DataRequest,
+    PipelineStep,
+    Trace,
+)
+
+
+class SplightDatalakeBaseModel(BaseModel):
+    timestamp: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc)
+    )
+    _collection_name: ClassVar[str] = "DatalakeModel"
+    _model_type: ClassVar[str] = ...
+
+    model_config = ConfigDict(extra="ignore")
+
+    @classmethod
+    def _get(
+        cls,
+        filters: dict[str, str],
+        extra_pipeline: list[dict[str, Any]] = [],
+        **params: dict,
+    ) -> list[Self]:
+        request = _to_data_request(
+            cls,
+            filters,
+            extra_pipeline,
+            **params,
+        )
+        return request.apply()
+
+    @classmethod
+    async def _async_get(
+        cls,
+        filters: dict[str, str],
+        extra_pipeline: list[dict[str, Any]] = [],
+        **params: dict,
+    ) -> list[Self]:
+        request = _to_data_request(cls, filters, extra_pipeline, **params)
+        instances = await request.async_apply()
+        return instances
+
+    @classmethod
+    def _get_dataframe(
+        cls,
+        filters: dict[str, str],
+        extra_pipeline: list[dict[str, Any]] = [],
+        **params: dict,
+    ) -> pd.DataFrame:
+        request = _to_data_request(
+            cls,
+            filters,
+            extra_pipeline,
+            **params,
+        )
+        instances = request.apply()
+        df = pd.DataFrame([instance.model_dump() for instance in instances])
+        if not df.empty:
+            df.index = df["timestamp"]
+            df.drop(columns="timestamp", inplace=True)
+        return df
+
+    def save(self) -> None:
+        records = self._to_record()
+        records.apply()
+
+    async def async_save(self) -> None:
+        records = self._to_record()
+        await records.async_apply()
+
+    @classmethod
+    def save_dataframe(cls, df: pd.DataFrame):
+        df = _fix_dataframe_timestamp(df)
+        instances = df.to_dict("records")
+        records = DataRecords(
+            collection=cls._collection_name,
+            records=instances,
+        )
+        records.apply()
+
+    def dict(self, *args, **kwargs) -> Dict:
+        d = super().model_dump(*args, **kwargs)
+        return {
+            k: v["id"] if isinstance(v, dict) and "id" in v.keys() else v
+            for k, v in d.items()
+        }
+
+    def _to_record(self) -> DataRecords:
+        return DataRecords(
+            collection=self._collection_name,
+            records=[self.model_dump(mode="json")],
+        )
+
+
+def _to_data_request(
+    model_class: TypeVar("T"),
+    filters: dict[str, str],
+    extra_pipeline: list[dict[str, Any]] = [],
+    **params: Dict,
+) -> DataRequest:
+    if not isinstance(extra_pipeline, list):
+        raise ValueError("extra_pipeline must be a list of dicts")
+    model_type = model_class._model_type
+    collection = (
+        "default" if model_type == "attribute_document" else "solutions"
+    )
+    request = DataRequest[model_class](
+        collection=collection,
+        from_timestamp=params.get("from_timestamp"),
+        to_timestamp=params.get("to_timestamp"),
+    )
+
+    if model_type == "attribute_document":
+        trace = Trace.from_address(**filters)
+    elif model_type == "solution_output_document":
+        trace = Trace.from_so_filter(**filters)
+    for step in extra_pipeline:
+        trace.add_step(PipelineStep.from_dict(step))
+
+    request.add_trace(trace)
+    return request
+
+
+def _fix_dataframe_timestamp(df: pd.DataFrame) -> pd.DataFrame:
+    if df["timestamp"][0].tz is None:
+        df["timestamp"] = df["timestamp"].apply(
+            lambda x: x.tz_localize(tz="UTC").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        )
+    else:
+        df["timestamp"] = df["timestamp"].apply(
+            lambda x: x.tz_convert(tz="UTC").strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        )
+    return df

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/splight_lib/models/_v4/native.py

@@ -14,7 +14,9 @@ class NativeOutput(SplightDatalakeBaseModel):
     asset: str | Asset
     attribute: str | Attribute
     output_format: str | None = None
+    _collection_name: ClassVar[Literal["default"]] = "default"
     _output_format: ClassVar[str] = "default"
+    _model_type: ClassVar[str] = "attribute_document"
 
     @field_validator("output_format", mode="before")
     def set_output_format(cls, v) -> str:
@@ -24,19 +26,23 @@ class NativeOutput(SplightDatalakeBaseModel):
     def get(
         cls, asset: str | Asset, attribute: str | Attribute, **params: dict
     ) -> list[Self]:
-        return super().get(asset, attribute, **params)
+        return super()._get({"asset": asset, "attribute": attribute}, **params)
 
     @classmethod
     async def async_get(
         cls, asset: str | Asset, attribute: str | Attribute, **params: dict
     ) -> list[Self]:
-        return await super().async_get(asset, attribute, **params)
+        return await super()._async_get(
+            {"asset": asset, "attribute": attribute}, **params
+        )
 
     @classmethod
     def get_dataframe(
         cls, asset: str | Asset, attribute: str | Attribute, **params: dict
     ) -> pd.DataFrame:
-        df = super().get_dataframe(asset, attribute, **params)
+        df = super()._get_dataframe(
+            {"asset": asset, "attribute": attribute}, **params
+        )
         df["output_format"] = cls._output_format
         return df
 
@@ -77,3 +83,56 @@ class Boolean(NativeOutput):
     value: bool
     output_format: Literal["Boolean"] = "Boolean"
     _output_format: ClassVar[str] = "Boolean"
+
+
+class SolutionOutputDocument(SplightDatalakeBaseModel):
+    asset: str | Asset
+    solution: str
+    output: str
+    value: bool | int | float | str
+
+    _collection_name: ClassVar[Literal["solutions"]] = "solutions"
+    _model_type: ClassVar[str] = "solution_output_document"
+
+    @classmethod
+    def get(
+        cls, solution: str, output: str, asset: str, **params: dict
+    ) -> list[Self]:
+        filters = {"solution": solution, "output": output, "asset": asset}
+        return super()._get(filters, **params)
+
+    @classmethod
+    async def async_get(
+        cls, solution: str, output: str, asset: str, **params: dict
+    ) -> list[Self]:
+        filters = {"solution": solution, "output": output, "asset": asset}
+        return await super()._async_get(filters, **params)
+
+    @classmethod
+    def get_dataframe(
+        cls, solution: str, output: str, asset: str, **params: dict
+    ) -> pd.DataFrame:
+        filters = {"solution": solution, "output": output, "asset": asset}
+        return super()._get_dataframe(filters, **params)
+
+    @classmethod
+    def latest(
+        cls,
+        solution: str,
+        output: str,
+        asset: str,
+        expiration: timedelta | None = None,
+    ) -> Self:
+        from_timestamp = None
+        if expiration:
+            from_timestamp = datetime.now(timezone.utc) - expiration
+        result = cls.get(
+            solution=solution,
+            output=output,
+            asset=asset,
+            from_timestamp=from_timestamp,
+            limit=1,
+        )
+        if result:
+            return result[0]
+        return None

{splight_lib-5.22.5.dev100 → splight_lib-5.23.0.dev0}/uv.lock

@@ -1748,7 +1748,7 @@ wheels = [
 
 [[package]]
 name = "splight-lib"
-version = "5.22.5"
+version = "5.23.0rc0"
 source = { editable = "." }
 dependencies = [
     { name = "apscheduler" },