sift-stack-py 0.3.2__py3-none-any.whl

sift_py/_internal/__init__.py

@@ -0,0 +1,5 @@
+"""
+INTERNAL MODULE
+
+Everything inside of this module is not meant to be used publicly.
+"""

sift_py/_internal/cel.py

@@ -0,0 +1,18 @@
+"""
+Utilities to interact with APIs that have a CEL-based interface.
+"""
+
+from typing import Iterable
+
+
+def cel_in(field: str, values: Iterable[str]) -> str:
+    """
+    Produces a list membership CEL expression. Example:
+
+    ```python
+    > print(cel_in("name", ["foo", "bar"]))
+    name in ["foo", "bar"]
+    ```
+    """
+    items = ",".join([f'"{val}"' for val in values])
+    return f"{field} in [{items}]"

sift_py/_internal/channel.py

@@ -0,0 +1,42 @@
+from typing import List, Optional, cast
+
+from sift.channels.v2.channels_pb2 import Channel as ChannelPb
+from sift.channels.v2.channels_pb2 import ListChannelsRequest, ListChannelsResponse
+from sift.channels.v2.channels_pb2_grpc import ChannelServiceStub
+
+
+def channel_fqn(name: str, component: Optional[str]) -> str:
+    return name if component is None or len(component) == 0 else f"{component}.{name}"
+
+
+def get_channels(
+    channel_service: ChannelServiceStub,
+    filter: str,
+    page_size: int = 1_000,
+    page_token: str = "",
+) -> List[ChannelPb]:
+    """
+    Queries all channels with the given filter. Filter must be a CEL expression.
+    """
+    channels_pb: List[ChannelPb] = []
+
+    req = ListChannelsRequest(
+        filter=filter,
+        page_size=page_size,
+        page_token=page_token,
+    )
+    res = cast(ListChannelsResponse, channel_service.ListChannels(req))
+    channels_pb.extend(res.channels)
+    next_page_token = res.next_page_token
+
+    while len(next_page_token) > 0:
+        req = ListChannelsRequest(
+            filter=filter,
+            page_size=page_size,
+            page_token=page_token,
+        )
+        res = cast(ListChannelsResponse, channel_service.ListChannels(req))
+        channels_pb.extend(res.channels)
+        next_page_token = res.next_page_token
+
+    return channels_pb

sift_py/_internal/convert/__init__.py

@@ -0,0 +1,3 @@
+"""
+Utilities for type-conversion and type-casting.
+"""

sift_py/_internal/convert/json.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class AsJson(ABC):
+    """
+    Utility sub-types that require custom-serialization meant to be used in conjunction with the
+    `to_json` function. Sub-types should implement `as_json` which should return the object that
+    you want passed to `json.dumps`.
+    """
+
+    @abstractmethod
+    def as_json(self) -> Any:
+        pass
+
+
+def to_json(value: Any) -> str:
+    """
+    Serializes `value` to a JSON string uses the `AsJson.as_json` implementation of the type.
+    """
+    return json.dumps(value, default=lambda x: x.as_json())

sift_py/_internal/convert/protobuf.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Generic, Type, TypeVar
+
+from google.protobuf.message import Message
+
+ProtobufMessage = Message
+
+T = TypeVar("T", bound=ProtobufMessage)
+
+
+class AsProtobuf(ABC, Generic[T]):
+    """
+    Abstract base class used to create create sub-types that can be treated
+    as an object that can be converted into an instance of `ProtobufMessage`.
+
+    If there are multiple possible protobuf targets then `as_pb` may be overloaded.
+    """
+
+    @abstractmethod
+    def as_pb(self, klass: Type[T]) -> T:
+        """
+        Performs the conversion into a sub-type of `ProtobufMessage`.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def from_pb(cls, message: T) -> T:
+        """
+        Converts a protobuf object to the type of the sub-class class.
+        """
+        pass

sift_py/_internal/convert/timestamp.py

@@ -0,0 +1,9 @@
+from datetime import datetime
+
+from google.protobuf.timestamp_pb2 import Timestamp
+
+
+def to_pb_timestamp(timestamp: datetime) -> Timestamp:
+    timestamp_pb = Timestamp()
+    timestamp_pb.FromDatetime(timestamp)
+    return timestamp_pb

sift_py/_internal/test_util/channel.py

@@ -0,0 +1,136 @@
+from collections.abc import AsyncIterable, Iterable
+from typing import Any, Callable, Optional, Union
+
+import grpc
+import grpc.aio as grpc_aio
+from grpc.aio import Channel as AsyncChannel
+from grpc_testing import Channel
+
+SerializingFunction = Callable[[Any], bytes]
+DeserializingFunction = Callable[[bytes], Any]
+DoneCallbackType = Callable[[Any], None]
+RequestIterableType = Union[Iterable, AsyncIterable]
+ResponseIterableType = AsyncIterable
+
+
+class MockChannel(Channel):
+    """
+    Used as a mock gRPC channel
+    """
+
+    def take_unary_unary(self, method_descriptor):
+        pass
+
+    def take_unary_stream(self, method_descriptor):
+        pass
+
+    def take_stream_unary(self, method_descriptor):
+        pass
+
+    def take_stream_stream(self, method_descriptor):
+        pass
+
+    def subscribe(self, callback, try_to_connect=False):
+        pass
+
+    def unsubscribe(self, callback):
+        pass
+
+    def unary_unary(
+        self,
+        method,
+        request_serializer=None,
+        response_deserializer=None,
+        _registered_method=False,
+    ):
+        pass
+
+    def unary_stream(
+        self,
+        method,
+        request_serializer=None,
+        response_deserializer=None,
+        _registered_method=False,
+    ):
+        pass
+
+    def stream_unary(
+        self,
+        method,
+        request_serializer=None,
+        response_deserializer=None,
+        _registered_method=False,
+    ):
+        pass
+
+    def stream_stream(
+        self,
+        method,
+        request_serializer=None,
+        response_deserializer=None,
+        _registered_method=False,
+    ):
+        pass
+
+    def close(self):
+        pass
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class MockAsyncChannel(AsyncChannel):
+    async def __aenter__(self):
+        pass
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    async def close(self, grace: Optional[float] = None):
+        pass
+
+    def get_state(self, try_to_connect: bool = False) -> grpc.ChannelConnectivity: ...
+
+    async def wait_for_state_change(
+        self,
+        last_observed_state: grpc.ChannelConnectivity,
+    ) -> None:
+        return None
+
+    async def channel_ready(self) -> None:
+        return None
+
+    def unary_unary(
+        self,
+        method: str,
+        request_serializer: Optional[SerializingFunction] = None,
+        response_deserializer: Optional[DeserializingFunction] = None,
+        _registered_method: Optional[bool] = False,
+    ) -> grpc_aio.UnaryUnaryMultiCallable: ...
+
+    def unary_stream(
+        self,
+        method: str,
+        request_serializer: Optional[SerializingFunction] = None,
+        response_deserializer: Optional[DeserializingFunction] = None,
+        _registered_method: Optional[bool] = False,
+    ) -> grpc_aio.UnaryStreamMultiCallable: ...
+
+    def stream_unary(
+        self,
+        method: str,
+        request_serializer: Optional[SerializingFunction] = None,
+        response_deserializer: Optional[DeserializingFunction] = None,
+        _registered_method: Optional[bool] = False,
+    ) -> grpc_aio.StreamUnaryMultiCallable: ...
+
+    def stream_stream(
+        self,
+        method: str,
+        request_serializer: Optional[SerializingFunction] = None,
+        response_deserializer: Optional[DeserializingFunction] = None,
+        _registered_method: Optional[bool] = False,
+    ) -> grpc_aio.StreamStreamMultiCallable: ...

sift_py/_internal/test_util/fn.py

@@ -0,0 +1,14 @@
+from types import ModuleType
+from typing import Callable
+
+
+def _mock_path(subject_module: ModuleType) -> Callable[[Callable], str]:
+    """
+    Returns a function that can be used to conveniently generate the mock path
+    for a function which could then be passed to `pytest_mock.MockFixture.patch`
+    """
+
+    def mock_fn(fn: Callable) -> str:
+        return f"{subject_module.__name__}.{fn.__name__}"
+
+    return mock_fn

sift_py/_internal/test_util/server_interceptor.py

@@ -0,0 +1,62 @@
+import abc
+from typing import Any, Callable, Optional, Tuple, cast
+
+import grpc
+
+
+class ServerInterceptor(grpc.ServerInterceptor, metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def intercept(
+        self,
+        method: Callable,
+        request_or_iterator: Any,
+        context: grpc.ServicerContext,
+        method_name: str,
+    ) -> Any:
+        return method(request_or_iterator, context)
+
+    def intercept_service(self, continuation, handler_call_details):
+        next_handler = continuation(handler_call_details)
+        if next_handler is None:
+            return
+
+        handler_factory, next_handler_method = _get_factory_and_method(next_handler)
+
+        def invoke_intercept_method(request_or_iterator, context):
+            method_name = handler_call_details.method
+            return self.intercept(
+                next_handler_method,
+                request_or_iterator,
+                context,
+                method_name,
+            )
+
+        return handler_factory(
+            invoke_intercept_method,
+            request_deserializer=next_handler.request_deserializer,
+            response_serializer=next_handler.response_serializer,
+        )
+
+
+class _RpcHandler(grpc.RpcMethodHandler):
+    unary_unary: Optional[Callable]
+    unary_stream: Optional[Callable]
+    stream_unary: Optional[Callable]
+    stream_stream: Optional[Callable]
+
+
+def _get_factory_and_method(
+    rpc_handler: grpc.RpcMethodHandler,
+) -> Tuple[Callable, Callable]:
+    handler = cast(_RpcHandler, rpc_handler)
+
+    if handler.unary_unary:
+        return grpc.unary_unary_rpc_method_handler, handler.unary_unary
+    elif handler.unary_stream:
+        return grpc.unary_stream_rpc_method_handler, handler.unary_stream
+    elif handler.stream_unary:
+        return grpc.stream_unary_rpc_method_handler, handler.stream_unary
+    elif handler.stream_stream:
+        return grpc.stream_stream_rpc_method_handler, handler.stream_stream
+    else:
+        raise Exception("Unreachable")

sift_py/_internal/time.py

@@ -0,0 +1,48 @@
+from datetime import datetime, timezone
+from typing import Union, cast
+
+import pandas as pd
+from google.protobuf.timestamp_pb2 import Timestamp as TimestampPb
+
+
+def to_timestamp_nanos(arg: Union[TimestampPb, pd.Timestamp, datetime, str, int]) -> pd.Timestamp:
+    """
+    Converts a variety of time-types to a pandas timestamp which supports nano-second precision.
+    """
+
+    if isinstance(arg, pd.Timestamp):
+        return arg
+    elif isinstance(arg, TimestampPb):
+        seconds = arg.seconds
+        nanos = arg.nanos
+
+        dt = datetime.fromtimestamp(seconds, tz=timezone.utc)
+        ts = pd.Timestamp(dt)
+
+        return cast(pd.Timestamp, ts + pd.Timedelta(nanos, unit="ns"))
+
+    elif isinstance(arg, int):
+        dt = datetime.fromtimestamp(arg, tz=timezone.utc)
+        return cast(pd.Timestamp, pd.Timestamp(dt))
+
+    else:
+        return cast(pd.Timestamp, pd.Timestamp(arg))
+
+
+def to_timestamp_pb(arg: Union[datetime, str, int]) -> TimestampPb:
+    """
+    Mainly used for testing at the moment. If using this for non-testing purposes
+    should probably make this more robust and support nano-second precision.
+    """
+
+    ts = TimestampPb()
+
+    if isinstance(arg, datetime):
+        ts.FromDatetime(arg)
+        return ts
+    elif isinstance(arg, int):
+        ts.FromDatetime(datetime.fromtimestamp(arg))
+        return ts
+    else:
+        ts.FromDatetime(datetime.fromisoformat(arg))
+        return ts

sift_py/_internal/user.py

@@ -0,0 +1,39 @@
+from typing import List, cast
+
+from sift.common.type.v1.user_pb2 import User
+from sift.users.v2.users_pb2 import ListActiveUsersRequest, ListActiveUsersResponse
+from sift.users.v2.users_pb2_grpc import UserServiceStub
+
+
+def get_active_users(
+    user_service: UserServiceStub,
+    filter: str,
+    page_size: int = 1_000,
+    page_token: str = "",
+) -> List[User]:
+    """
+    Get active users from the user service with the given filter.
+    The filter must be a CEL expression.
+    """
+    users_pb: List[User] = []
+
+    req = ListActiveUsersRequest(
+        filter=filter,
+        page_size=page_size,
+        page_token=page_token,
+    )
+    res = cast(ListActiveUsersResponse, user_service.ListActiveUsers(req))
+    users_pb.extend(res.users)
+    next_page_token = res.next_page_token
+
+    while len(next_page_token) > 0:
+        req = ListActiveUsersRequest(
+            filter=filter,
+            page_size=page_size,
+            page_token=page_token,
+        )
+        res = cast(ListActiveUsersResponse, user_service.ListActiveUsers(req))
+        users_pb.extend(res.users)
+        next_page_token = res.next_page_token
+
+    return users_pb

sift_py/data/__init__.py

@@ -0,0 +1,171 @@
+"""
+This module contains tools to download telemetry from the Sift data API. The
+core component of this module is the `sift_py.data.service.DataService` and the
+`sift_py.data.query` module. The former is what's used to execute a data query,
+while the latter is what's used to actually construct the query. A typical query could look
+something like this:
+
+```python
+query = DataQuery(
+    asset_name="NostromoLV426",
+    start_time="2024-07-04T18:09:08.555-07:00",
+    end_time="2024-07-04T18:09:11.556-07:00",
+    sample_ms=16,
+    channels=[
+        ChannelQuery(
+            channel_name="voltage",
+            run_name="[NostromoLV426].1720141748.047512"
+        ),
+        ChannelQuery(
+            channel_name="velocity",
+            component="mainmotors",
+            run_name="[NostromoLV426].1720141748.047512",
+        ),
+        ChannelQuery(
+            channel_name="gpio",
+            run_name="[NostromoLV426].1720141748.047512",
+        ),
+    ],
+)
+```
+
+This query, once passed to the `sift_py.data.service.DataService.execute` method, will
+fetch data between `start_time` and `end_time` at the sampling rate given by `sample_ms`.
+
+> ⚠️ **Warning**: Note on Performance
+>
+> Currently the results of a query are all buffered in memory, so it it best to be mindful
+> about your memory limitations and overall performance requirements when requesting data
+> within a large time range and a slow sampling rate. Full-fidelity data is returned
+> when the `sample_ms` is set to `0`.
+
+The data API allows you to download telemetry for both channels as well as calculated
+channels. The following examples demonstrate how to download data for both channels and
+calculated channels, respectively.
+
+* [Regular Channels](#regular-channels)
+* [Calculated Channels](#calculated-channels)
+
+## Regular Channels
+
+```python
+import asyncio
+import functools
+import pandas as pd
+from sift_py.data.query import ChannelQuery, DataQuery
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_async_channel
+from sift_py.data.service import DataService
+
+
+async def channel_demo():
+    channel_config: SiftChannelConfig = {
+        "apikey": "my-key"
+        "uri": "sift-uri"
+    }
+
+    async with use_sift_async_channel(channel_config) as channel:
+        data_service = DataService(channel)
+
+        query = DataQuery(
+            asset_name="NostromoLV426",
+            start_time="2024-07-04T18:09:08.555-07:00",
+            end_time="2024-07-04T18:09:11.556-07:00",
+            channels=[
+                ChannelQuery(
+                    channel_name="voltage",
+                    run_name="[NostromoLV426].1720141748.047512"
+                ),
+                ChannelQuery(
+                    channel_name="velocity",
+                    component="mainmotors",
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+                ChannelQuery(
+                    channel_name="gpio",
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+            ],
+        )
+
+        result = await data_service.execute(query)
+
+        data_frames = [
+            pd.DataFrame(data.columns())
+            for data in result.channels("voltage", "mainmotors.velocity", "gpio.12v")
+        ]
+
+        merged_frame = functools.reduce(
+            lambda x, y: pd.merge_asof(x, y, on="time"), data_frames
+        )
+
+        merged_frame.to_csv("my_csv.csv")
+
+if __name__ == "__main__":
+    asyncio.run(example())
+```
+
+## Calculated Channels
+
+```python
+import asyncio
+import functools
+import pandas as pd
+from sift_py.data.query import ChannelQuery, DataQuery
+from sift_py.grpc.transport import SiftChannelConfig, use_sift_async_channel
+from sift_py.data.service import DataService
+
+
+async def channel_demo():
+    channel_config: SiftChannelConfig = {
+        "apikey": "my-key"
+        "uri": "sift-uri"
+    }
+
+    async with use_sift_async_channel(channel_config) as channel:
+        data_service = DataService(channel)
+
+        query = DataQuery(
+            asset_name="NostromoLV426",
+            start_time="2024-07-04T18:09:08.555-07:00",
+            end_time="2024-07-04T18:09:11.556-07:00",
+            channels=[
+                CalculatedChannelQuery(
+                    channel_key="calc-voltage",
+                    expression="$1 + 10",
+                    expression_channel_references=[
+                        {
+                            "reference": "$1",
+                            "channel_name": "voltage",
+                        },
+                    ],
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+                CalculatedChannelQuery(
+                    channel_key="calc-velocity",
+                    expression="$1 * 2",
+                    expression_channel_references=[
+                        {
+                            "reference": "$1",
+                            "channel_name": "velocity",
+                            "component": "mainmotors",
+                        },
+                    ],
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+            ],
+        )
+
+        result = await data_service.execute(query)
+        calc_voltage, calc_velocity = result.channels("calc-voltage", "calc-velocity")
+
+        calc_voltage_df = pd.DataFrame(calc_voltage.columns())
+        calc_velocity_df = pd.DataFrame(calc_velocity.columns())
+
+        merged_frame = pd.merge_asof(calc_voltage_df, calc_velocity_df, on="time")
+
+        merged_frame.to_csv("my_csv.csv")
+
+if __name__ == "__main__":
+    asyncio.run(example())
+```
+"""

sift_py/data/_channel.py

@@ -0,0 +1,38 @@
+from typing import Any, List
+
+import pandas as pd
+
+from sift_py.ingestion.channel import ChannelDataType
+
+
+class ChannelTimeSeries:
+    data_type: ChannelDataType
+    time_column: List[pd.Timestamp]
+    value_column: List[Any]
+
+    def __init__(
+        self,
+        data_type: ChannelDataType,
+        time_column: List[pd.Timestamp],
+        value_column: List[Any],
+    ):
+        if len(time_column) != len(value_column):
+            raise Exception("Both arguments, `time_column` and `value_column` must equal lengths.")
+
+        self.data_type = data_type
+        self.time_column = time_column
+        self.value_column = value_column
+
+    def sort_time_series(self):
+        points = [(t, v) for t, v in zip(self.time_column, self.value_column)]
+        points.sort(key=lambda x: x[0])
+
+        time_column = []
+        value_column = []
+
+        for ts, val in points:
+            time_column.append(ts)
+            value_column.append(val)
+
+        self.time_column = time_column
+        self.value_column = value_column