sift-stack-py 0.3.2__py3-none-any.whl

sift_py/data/_deserialize.py

@@ -0,0 +1,208 @@
+from copy import deepcopy
+from enum import Enum
+from typing import List, Tuple, cast
+
+from google.protobuf.any_pb2 import Any
+from sift.data.v1.data_pb2 import (
+    BitFieldValues,
+    BoolValues,
+    DoubleValues,
+    EnumValues,
+    FloatValues,
+    Int32Values,
+    Int64Values,
+    Metadata,
+    StringValues,
+    Uint32Values,
+    Uint64Values,
+)
+
+from sift_py._internal.time import to_timestamp_nanos
+from sift_py.data._channel import ChannelTimeSeries
+from sift_py.error import SiftError
+from sift_py.ingestion.channel import ChannelDataType
+
+
+class ChannelValues(Enum):
+    DOUBLE_VALUES = "sift.data.v1.DoubleValues"
+    FLOAT_VALUES = "sift.data.v1.FloatValues"
+    STRING_VALUES = "sift.data.v1.StringValues"
+    ENUM_VALUES = "sift.data.v1.EnumValues"
+    BIT_FIELD_VALUES = "sift.data.v1.BitFieldValues"
+    BOOL_VALUES = "sift.data.v1.BoolValues"
+    INT32_VALUES = "sift.data.v1.Int32Values"
+    INT64_VALUES = "sift.data.v1.Int64Values"
+    UINT32_VALUES = "sift.data.v1.Uint32Values"
+    UINT64_VALUES = "sift.data.v1.Uint64Values"
+
+
+def try_deserialize_channel_data(channel_values: Any) -> List[Tuple[Metadata, ChannelTimeSeries]]:
+    if ChannelValues.DOUBLE_VALUES.value in channel_values.type_url:
+        double_values = cast(DoubleValues, DoubleValues.FromString(channel_values.value))
+        metadata = double_values.metadata
+
+        time_column = []
+        double_value_column = []
+
+        for v in double_values.values:
+            time_column.append(to_timestamp_nanos(v.timestamp))
+            double_value_column.append(v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, double_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.FLOAT_VALUES.value in channel_values.type_url:
+        float_values = cast(FloatValues, FloatValues.FromString(channel_values.value))
+        metadata = float_values.metadata
+
+        time_column = []
+        float_value_column = []
+
+        for float_v in float_values.values:
+            time_column.append(to_timestamp_nanos(float_v.timestamp))
+            float_value_column.append(float_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, float_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.STRING_VALUES.value in channel_values.type_url:
+        string_values = cast(StringValues, StringValues.FromString(channel_values.value))
+        metadata = string_values.metadata
+
+        time_column = []
+        string_value_column = []
+
+        for string_v in string_values.values:
+            time_column.append(to_timestamp_nanos(string_v.timestamp))
+            string_value_column.append(string_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, string_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.ENUM_VALUES.value in channel_values.type_url:
+        enum_values = cast(EnumValues, EnumValues.FromString(channel_values.value))
+        metadata = enum_values.metadata
+
+        time_column = []
+        enum_value_column = []
+
+        for enum_v in enum_values.values:
+            time_column.append(to_timestamp_nanos(enum_v.timestamp))
+            enum_value_column.append(enum_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, enum_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.BOOL_VALUES.value in channel_values.type_url:
+        bool_values = cast(BoolValues, BoolValues.FromString(channel_values.value))
+        metadata = bool_values.metadata
+
+        time_column = []
+        bool_value_column = []
+
+        for bool_v in bool_values.values:
+            time_column.append(to_timestamp_nanos(bool_v.timestamp))
+            bool_value_column.append(bool_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, bool_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.INT32_VALUES.value in channel_values.type_url:
+        int32_values = cast(Int32Values, Int32Values.FromString(channel_values.value))
+        metadata = int32_values.metadata
+
+        time_column = []
+        int32_value_column = []
+
+        for int32_v in int32_values.values:
+            time_column.append(to_timestamp_nanos(int32_v.timestamp))
+            int32_value_column.append(int32_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, int32_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.INT64_VALUES.value in channel_values.type_url:
+        int64_values = cast(Int64Values, Int64Values.FromString(channel_values.value))
+        metadata = int64_values.metadata
+
+        time_column = []
+        int64_value_column = []
+
+        for int64_v in int64_values.values:
+            time_column.append(to_timestamp_nanos(int64_v.timestamp))
+            int64_value_column.append(int64_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, int64_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.UINT32_VALUES.value in channel_values.type_url:
+        uint32_values = cast(Uint32Values, Uint32Values.FromString(channel_values.value))
+        metadata = uint32_values.metadata
+
+        time_column = []
+        uint32_value_column = []
+
+        for uint32_v in uint32_values.values:
+            time_column.append(to_timestamp_nanos(uint32_v.timestamp))
+            uint32_value_column.append(uint32_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, uint32_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.UINT64_VALUES.value in channel_values.type_url:
+        uint64_values = cast(Uint64Values, Uint64Values.FromString(channel_values.value))
+        metadata = uint64_values.metadata
+
+        time_column = []
+        uint64_value_column = []
+
+        for uint64_v in uint64_values.values:
+            time_column.append(to_timestamp_nanos(uint64_v.timestamp))
+            uint64_value_column.append(uint64_v.value)
+
+        time_series = ChannelTimeSeries(
+            ChannelDataType.from_pb(metadata.data_type), time_column, uint64_value_column
+        )
+        return [(metadata, time_series)]
+
+    elif ChannelValues.BIT_FIELD_VALUES.value in channel_values.type_url:
+        bit_field_values = cast(BitFieldValues, BitFieldValues.FromString(channel_values.value))
+        metadata = bit_field_values.metadata
+        data_type = ChannelDataType.from_pb(metadata.data_type)
+        channel_name = metadata.channel.name
+
+        parsed_data: List[Tuple[Metadata, ChannelTimeSeries]] = []
+
+        for bit_field_element in bit_field_values.values:
+            md_copy = deepcopy(bit_field_values.metadata)
+            md_copy.channel.name = f"{channel_name}.{bit_field_element.name}"
+
+            time_column = []
+            bit_field_el_column = []
+
+            for bf_v in bit_field_element.values:
+                time_column.append(to_timestamp_nanos(bf_v.timestamp))
+                bit_field_el_column.append(bf_v.value)
+
+            time_series = ChannelTimeSeries(data_type, time_column, bit_field_el_column)
+            parsed_data.append((md_copy, time_series))
+
+        return parsed_data
+
+    raise SiftError(f"Received an unknown channel-type '{channel_values.type_url}'.")

sift_py/data/_deserialize_test.py

@@ -0,0 +1,134 @@
+from google.protobuf.any_pb2 import Any
+from sift.common.type.v1.channel_bit_field_element_pb2 import ChannelBitFieldElement
+from sift.common.type.v1.channel_data_type_pb2 import (
+    CHANNEL_DATA_TYPE_BIT_FIELD,
+    CHANNEL_DATA_TYPE_DOUBLE,
+)
+from sift.data.v1.data_pb2 import (
+    BitFieldElementValues,
+    BitFieldValue,
+    BitFieldValues,
+    DoubleValue,
+    DoubleValues,
+    Metadata,
+)
+
+from sift_py._internal.time import to_timestamp_nanos, to_timestamp_pb
+from sift_py.data._deserialize import try_deserialize_channel_data
+
+
+def test_try_deserialize_channel_data_double():
+    metadata = Metadata(
+        data_type=CHANNEL_DATA_TYPE_DOUBLE, channel=Metadata.Channel(name="double-channel")
+    )
+
+    time_a = "2024-07-04T18:09:08.555-07:00"
+    time_b = "2024-07-04T18:09:09.555-07:00"
+
+    double_values = DoubleValues(
+        metadata=metadata,
+        values=[
+            DoubleValue(
+                timestamp=to_timestamp_pb(time_a),
+                value=10,
+            ),
+            DoubleValue(
+                timestamp=to_timestamp_pb(time_b),
+                value=11,
+            ),
+        ],
+    )
+
+    raw_values = Any()
+    raw_values.Pack(double_values)
+
+    deserialized_data = try_deserialize_channel_data(raw_values)
+
+    assert len(deserialized_data) == 1
+
+    metadata, time_series = deserialized_data[0]
+
+    assert metadata.data_type == CHANNEL_DATA_TYPE_DOUBLE
+    assert metadata.channel.name == "double-channel"
+    assert len(time_series.time_column) == 2
+    assert len(time_series.value_column) == 2
+    assert time_series.value_column[0] == 10
+    assert time_series.value_column[1] == 11
+    assert time_series.time_column[0] == to_timestamp_nanos(time_a)
+    assert time_series.time_column[1] == to_timestamp_nanos(time_b)
+
+
+def test_try_deserialize_channel_data_bit_field_elements():
+    metadata = Metadata(
+        data_type=CHANNEL_DATA_TYPE_BIT_FIELD,
+        channel=Metadata.Channel(
+            name="gpio",
+            bit_field_elements=[
+                ChannelBitFieldElement(
+                    name="12v",
+                    index=0,
+                    bit_count=4,
+                ),
+                ChannelBitFieldElement(
+                    name="heater",
+                    index=4,
+                    bit_count=4,
+                ),
+            ],
+        ),
+    )
+
+    time_a = "2024-07-04T18:09:08.555-07:00"
+
+    value_a = int("10000001", 2)
+    value_b = int("11110001", 2)
+
+    bit_field_values = BitFieldValues(
+        metadata=metadata,
+        values=[
+            BitFieldElementValues(
+                name="12v",
+                values=[
+                    BitFieldValue(
+                        timestamp=to_timestamp_pb(time_a),
+                        value=value_a,
+                    )
+                ],
+            ),
+            BitFieldElementValues(
+                name="heater",
+                values=[
+                    BitFieldValue(
+                        timestamp=to_timestamp_pb(time_a),
+                        value=value_b,
+                    )
+                ],
+            ),
+        ],
+    )
+
+    raw_values = Any()
+    raw_values.Pack(bit_field_values)
+
+    deserialized_data = try_deserialize_channel_data(raw_values)
+
+    assert len(deserialized_data) == 2
+
+    metadata_12v, time_series_12v = deserialized_data[0]
+    metadata_heater, time_series_heater = deserialized_data[1]
+
+    assert metadata_12v.data_type == CHANNEL_DATA_TYPE_BIT_FIELD
+    assert metadata_heater.data_type == CHANNEL_DATA_TYPE_BIT_FIELD
+
+    assert metadata_12v.channel.name == "gpio.12v"
+    assert metadata_heater.channel.name == "gpio.heater"
+
+    assert len(time_series_12v.time_column) == 1
+    assert len(time_series_12v.value_column) == 1
+    assert len(time_series_heater.time_column) == 1
+    assert len(time_series_heater.value_column) == 1
+
+    assert time_series_12v.value_column[0] == value_a
+    assert time_series_heater.value_column[0] == value_b
+    assert time_series_12v.time_column[0] == to_timestamp_nanos(time_a)
+    assert time_series_heater.time_column[0] == to_timestamp_nanos(time_a)

sift_py/data/_service_test.py

@@ -0,0 +1,276 @@
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Iterator
+
+import pytest
+from google.protobuf.any_pb2 import Any
+from pytest_mock import MockFixture, MockType
+from sift.assets.v1.assets_pb2 import Asset
+from sift.channels.v2.channels_pb2 import Channel
+from sift.common.type.v1.channel_bit_field_element_pb2 import ChannelBitFieldElement
+from sift.common.type.v1.channel_data_type_pb2 import (
+    CHANNEL_DATA_TYPE_BIT_FIELD,
+    CHANNEL_DATA_TYPE_DOUBLE,
+)
+from sift.data.v1.data_pb2 import (
+    BitFieldElementValues,
+    BitFieldValue,
+    BitFieldValues,
+    DoubleValue,
+    DoubleValues,
+    Metadata,
+)
+from sift.runs.v2.runs_pb2 import Run
+
+from sift_py._internal.test_util.channel import MockAsyncChannel
+from sift_py._internal.time import to_timestamp_pb
+from sift_py.data.query import ChannelQuery, DataQuery
+from sift_py.data.service import DataService
+
+
+@pytest.mark.asyncio
+async def test_data_service_execute_regular_channels(mocker: MockFixture):
+    with patch_grpc_calls_channels(mocker) as mocks:
+        channel = MockAsyncChannel()
+        data_service = DataService(channel)
+
+        start_time = datetime.now(timezone.utc)
+        end_time = start_time + timedelta(minutes=2)
+
+        query = DataQuery(
+            asset_name="NostromoLV428",
+            start_time=start_time,
+            end_time=end_time,
+            sample_ms=0,
+            channels=[
+                ChannelQuery(
+                    channel_name="velocity",
+                    component="mainmotor",
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+                ChannelQuery(
+                    channel_name="gpio",
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+                ChannelQuery(
+                    channel_name="valve.pressure",
+                    run_name="[NostromoLV426].1720141748.047512",
+                ),
+            ],
+        )
+
+        result = await data_service.execute(query)
+
+        mock_get_asset = mocks["mock_get_asset_by_name"]
+        mock_get_channels = mocks["mock_get_channels_by_asset_id"]
+        mock_get_runs = mocks["mock_get_runs_by_names"]
+
+        mock_get_asset.assert_called_once()
+        mock_get_channels.assert_called_once()
+        mock_get_runs.assert_called_once()
+
+        # bit field elements count as separate channels
+        assert len(result.all_channels()) == 4
+        assert not result.channel("velocity")
+        assert not result.channels("velocity")
+        assert len(result.channels("mainmotor.velocity")) == 1
+
+        velocity = result.channel("mainmotor.velocity")
+        assert velocity is not None
+        assert len(velocity.timestamps) == 2
+        assert len(velocity.time_column()["time"]) == 2
+        assert len(velocity.time_column("custom_column_name")["custom_column_name"]) == 2
+        assert len(velocity.value_column()["mainmotor.velocity"]) == 2
+        assert len(velocity.value_column("custom_column_name")["custom_column_name"]) == 2
+
+        all_columns = velocity.columns()
+        assert len(all_columns) == 2
+        assert len(all_columns["time"]) == 2
+        assert len(all_columns["mainmotor.velocity"]) == 2
+
+        all_columns_custom = velocity.columns(
+            time_column_name="ts",
+            value_column_name="velocity",
+        )
+        assert len(all_columns_custom) == 2
+        assert len(all_columns_custom["ts"]) == 2
+        assert len(all_columns_custom["velocity"]) == 2
+
+        gpio = result.channel("gpio")
+        assert not gpio
+
+        gpio_12v = result.channel("gpio.12v")
+        assert gpio_12v is not None
+        assert len(gpio_12v.timestamps) == 1
+        assert len(gpio_12v.time_column()["time"]) == 1
+        assert len(gpio_12v.time_column("custom_column_name")["custom_column_name"]) == 1
+        assert len(gpio_12v.value_column()["gpio.12v"]) == 1
+        assert len(gpio_12v.value_column("12v")["12v"]) == 1
+
+        gpio_heater = result.channel("gpio.heater")
+        assert gpio_heater is not None
+        assert len(gpio_heater.timestamps) == 1
+        assert len(gpio_heater.time_column()["time"]) == 1
+        assert len(gpio_heater.time_column("custom_column_name")["custom_column_name"]) == 1
+        assert len(gpio_heater.value_column()["gpio.heater"]) == 1
+        assert len(gpio_heater.value_column("heater")["heater"]) == 1
+
+        pressure = result.channel("valve.pressure")
+        assert pressure is not None
+        assert len(pressure.timestamps) == 2
+        assert len(pressure.time_column()["time"]) == 2
+        assert len(pressure.time_column("custom_column_name")["custom_column_name"]) == 2
+        assert len(pressure.value_column()["valve.pressure"]) == 2
+        assert len(pressure.value_column("custom_column_name")["custom_column_name"]) == 2
+
+        all_columns = pressure.columns()
+        assert len(all_columns) == 2
+        assert len(all_columns["time"]) == 2
+        assert len(all_columns["valve.pressure"]) == 2
+
+        all_columns_custom = pressure.columns(
+            time_column_name="ts",
+            value_column_name="valve.pressure",
+        )
+        assert len(all_columns_custom) == 2
+        assert len(all_columns_custom["ts"]) == 2
+        assert len(all_columns_custom["valve.pressure"]) == 2
+
+
+@contextmanager
+def patch_grpc_calls_channels(mocker: MockFixture) -> Iterator[Dict[str, MockType]]:
+    mock__get_asset_by_name = mocker.patch.object(DataService, "_get_asset_by_name")
+    mock__get_asset_by_name.return_value = Asset(
+        asset_id="b7955799-9893-4acf-bf14-50052284020c", name="NostromoLV428"
+    )
+
+    mock__get_channels_by_asset_id = mocker.patch.object(DataService, "_get_channels_by_asset_id")
+    mock__get_channels_by_asset_id.return_value = [
+        Channel(
+            channel_id="e8662647-12f7-465f-85dc-cb02513944e0",
+            name="velocity",
+            component="mainmotor",
+            data_type=CHANNEL_DATA_TYPE_DOUBLE,
+        ),
+        Channel(
+            channel_id="97e25141-ed3e-4538-b063-c3eac30838ce",
+            name="gpio",
+            data_type=CHANNEL_DATA_TYPE_BIT_FIELD,
+        ),
+        Channel(
+            channel_id="87e25141-ed3e-4538-b063-c3eac30838cd",
+            name="valve.pressure",
+            data_type=CHANNEL_DATA_TYPE_DOUBLE,
+        ),
+    ]
+
+    mock__get_runs_by_names = mocker.patch.object(DataService, "_get_runs_by_names")
+    mock__get_runs_by_names.return_value = [
+        Run(
+            run_id="9b7f6c5f-cabc-4481-b048-6f12fc6b5b68",
+            name="[NostromoLV426].1720141748.047512",
+        )
+    ]
+
+    time_a = "2024-07-04T18:09:08.555-07:00"
+    time_b = "2024-07-04T18:09:09.555-07:00"
+
+    velocity_values = DoubleValues(
+        metadata=Metadata(
+            data_type=CHANNEL_DATA_TYPE_DOUBLE,
+            channel=Metadata.Channel(name="velocity", component="mainmotor"),
+        ),
+        values=[
+            DoubleValue(
+                timestamp=to_timestamp_pb(time_a),
+                value=10,
+            ),
+            DoubleValue(
+                timestamp=to_timestamp_pb(time_b),
+                value=11,
+            ),
+        ],
+    )
+
+    raw_velocity_values = Any()
+    raw_velocity_values.Pack(velocity_values)
+
+    time_a = "2024-07-04T18:09:08.555-07:00"
+    time_b = "2024-07-04T18:09:09.555-07:00"
+
+    pressure_values = DoubleValues(
+        metadata=Metadata(
+            data_type=CHANNEL_DATA_TYPE_DOUBLE,
+            channel=Metadata.Channel(name="valve.pressure"),
+        ),
+        values=[
+            DoubleValue(
+                timestamp=to_timestamp_pb(time_a),
+                value=10,
+            ),
+            DoubleValue(
+                timestamp=to_timestamp_pb(time_b),
+                value=11,
+            ),
+        ],
+    )
+
+    raw_pressure_values = Any()
+    raw_pressure_values.Pack(pressure_values)
+
+    bit_field_values = BitFieldValues(
+        metadata=Metadata(
+            data_type=CHANNEL_DATA_TYPE_BIT_FIELD,
+            channel=Metadata.Channel(
+                name="gpio",
+                bit_field_elements=[
+                    ChannelBitFieldElement(
+                        name="12v",
+                        index=0,
+                        bit_count=4,
+                    ),
+                    ChannelBitFieldElement(
+                        name="heater",
+                        index=4,
+                        bit_count=4,
+                    ),
+                ],
+            ),
+        ),
+        values=[
+            BitFieldElementValues(
+                name="12v",
+                values=[
+                    BitFieldValue(
+                        timestamp=to_timestamp_pb(time_a),
+                        value=int("10000001", 2),
+                    )
+                ],
+            ),
+            BitFieldElementValues(
+                name="heater",
+                values=[
+                    BitFieldValue(
+                        timestamp=to_timestamp_pb(time_a),
+                        value=int("11110001", 2),
+                    )
+                ],
+            ),
+        ],
+    )
+
+    raw_bit_field_values = Any()
+    raw_bit_field_values.Pack(bit_field_values)
+
+    mock__get_data = mocker.patch.object(DataService, "_get_data")
+    mock__get_data.side_effect = [
+        [[raw_velocity_values]],
+        [[raw_bit_field_values]],
+        [[raw_pressure_values]],
+    ]
+    yield {
+        "mock_get_asset_by_name": mock__get_asset_by_name,
+        "mock_get_runs_by_names": mock__get_runs_by_names,
+        "mock_get_channels_by_asset_id": mock__get_channels_by_asset_id,
+        "mock_get_data": mock__get_data,
+    }

sift_py/data/_validate.py

@@ -0,0 +1,10 @@
+import re
+
+CHANNEL_REFERENCE_REGEX = re.compile(r"^\$\d+$")
+
+
+def validate_channel_reference(ref: str):
+    if CHANNEL_REFERENCE_REGEX.match(ref) is None:
+        raise ValueError(
+            f"Invalid channel reference key '{ref}'. Expected an integer prefixed with '$' e.g. '$1', '$2', and so on."
+        )

sift_py/data/error.py

@@ -0,0 +1,5 @@
+class DataError(Exception):
+    msg: str
+
+    def __init__(self, msg: str):
+        super().__init__(msg)