sift-stack-py 0.3.2__py3-none-any.whl

sift_py/ingestion/config/telemetry.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional, cast
+
+from typing_extensions import Self
+
+from sift_py._internal.channel import channel_fqn
+from sift_py.ingestion.channel import (
+    ChannelBitFieldElement,
+    ChannelConfig,
+    ChannelDataType,
+    ChannelEnumType,
+)
+from sift_py.ingestion.config.yaml.load import (
+    load_named_expression_modules,
+    read_and_validate,
+)
+from sift_py.ingestion.config.yaml.spec import TelemetryConfigYamlSpec
+from sift_py.ingestion.flow import FlowConfig
+from sift_py.rule.config import (
+    ExpressionChannelReference,
+    ExpressionChannelReferenceChannelConfig,
+    RuleAction,
+    RuleActionAnnotationKind,
+    RuleActionCreateDataReviewAnnotation,
+    RuleActionCreatePhaseAnnotation,
+    RuleConfig,
+)
+from sift_py.yaml.rule import RuleYamlSpec, load_rule_modules
+
+
+class TelemetryConfig:
+    """
+    Configurations necessary to start ingestion.
+    - `asset_name`: The name of the asset that you wish to telemeter data for.
+    - `ingestion_client_key`: An arbitrary string chosen by the user to uniquely identify this ingestion configuration.
+    - `flows`: A single flow can specify a single channel value or a set of channel values that are ingested together.
+    - `organization_id`: ID of your organization in Sift. This field is only required if your user belongs to multiple organizations.
+    - `rules`: Rules to evaluate during ingestion.
+    """
+
+    asset_name: str
+    ingestion_client_key: str
+    organization_id: Optional[str]
+    flows: List[FlowConfig]
+    rules: List[RuleConfig]
+
+    def __init__(
+        self,
+        asset_name: str,
+        ingestion_client_key: str,
+        organization_id: Optional[str] = None,
+        flows: List[FlowConfig] = [],
+        rules: List[RuleConfig] = [],
+    ):
+        """
+        Will raise a `TelemetryConfigValidationError` under the following conditions:
+        - Multiple flows with the same name
+        - Multiple rules with the same name
+        - Identical channels in the same flow
+        """
+        self.__class__.validate_flows(flows)
+        self.__class__.validate_rules(rules)
+
+        self.asset_name = asset_name
+        self.ingestion_client_key = ingestion_client_key
+        self.organization_id = organization_id
+        self.flows = flows
+        self.rules = rules
+
+    @staticmethod
+    def validate_rules(rules: List[RuleConfig]):
+        """
+        Ensure that there are no rules with identical names
+        """
+        seen_rule_names = set()
+
+        for rule in rules:
+            if rule.name in seen_rule_names:
+                raise TelemetryConfigValidationError(
+                    f"Can't have two rules with identical names, '{rule.name}'."
+                )
+            seen_rule_names.add(rule.name)
+
+    @staticmethod
+    def validate_flows(flows: List[FlowConfig]):
+        """
+        Ensures no duplicate channels and flows with the same name, otherwise raises a `TelemetryConfigValidationError` exception.
+        """
+        flow_names = set()
+
+        for flow in flows:
+            seen_channels = set()
+
+            if flow.name in flow_names:
+                raise TelemetryConfigValidationError(
+                    f"Can't have two flows with the same name, '{flow.name}'."
+                )
+
+            flow_names.add(flow.name)
+
+            for channel in flow.channels:
+                fqn = channel.fqn()
+
+                if fqn in seen_channels:
+                    raise TelemetryConfigValidationError(
+                        f"Can't have two identical channels, '{fqn}', in flow '{flow.name}'."
+                    )
+                else:
+                    seen_channels.add(fqn)
+
+    @classmethod
+    def try_from_yaml(
+        cls,
+        path: Path,
+        named_expression_modules: Optional[List[Path]] = None,
+        named_rule_modules: Optional[List[Path]] = None,
+    ) -> Self:
+        """
+        Initializes a telemetry config from a YAML file found at the provided `path` as well as optional
+        paths to named expression modules if named expressions are leveraged.
+        """
+
+        config_as_yaml = read_and_validate(path)
+
+        named_expressions = {}
+        rule_modules = []
+        if named_expression_modules is not None:
+            named_expressions = load_named_expression_modules(named_expression_modules)
+        if named_rule_modules is not None:
+            rule_modules = load_rule_modules(named_rule_modules)
+
+        return cls._from_yaml(config_as_yaml, named_expressions, rule_modules)
+
+    @classmethod
+    def _from_yaml(
+        cls,
+        config_as_yaml: TelemetryConfigYamlSpec,
+        named_expressions: Dict[str, str] = {},
+        rule_modules: List[RuleYamlSpec] = [],
+    ) -> Self:
+        rules = []
+        flows = []
+
+        for flow in config_as_yaml.get("flows", []):
+            channels = []
+
+            for channel in flow["channels"]:
+                data_type = cast(ChannelDataType, ChannelDataType.from_str(channel["data_type"]))
+
+                bit_field_elements = []
+                for bit_field_element in channel.get("bit_field_elements", []):
+                    bit_field_elements.append(
+                        ChannelBitFieldElement(
+                            name=bit_field_element["name"],
+                            index=bit_field_element["index"],
+                            bit_count=bit_field_element["bit_count"],
+                        )
+                    )
+
+                enum_types = []
+                for enum_type in channel.get("enum_types", []):
+                    enum_types.append(
+                        ChannelEnumType(
+                            name=enum_type["name"],
+                            key=enum_type["key"],
+                        )
+                    )
+
+                channels.append(
+                    ChannelConfig(
+                        name=channel["name"],
+                        data_type=data_type,
+                        description=channel.get("description"),
+                        unit=channel.get("unit"),
+                        component=channel.get("component"),
+                        bit_field_elements=bit_field_elements,
+                        enum_types=enum_types,
+                    )
+                )
+
+            flows.append(
+                FlowConfig(
+                    name=flow["name"],
+                    channels=channels,
+                )
+            )
+
+        yaml_rules = config_as_yaml.get("rules", []) + rule_modules
+
+        for rule in yaml_rules:
+            action: Optional[RuleAction] = None
+            description: str = ""
+            annotation_type = RuleActionAnnotationKind.from_str(rule["type"])
+            tags = rule.get("tags")
+            description = rule.get("description", "")
+
+            action = RuleActionCreatePhaseAnnotation(tags)
+            if annotation_type == RuleActionAnnotationKind.REVIEW:
+                action = RuleActionCreateDataReviewAnnotation(
+                    assignee=rule.get("assignee"),
+                    tags=tags,
+                )
+
+            channel_references: List[
+                ExpressionChannelReference | ExpressionChannelReferenceChannelConfig
+            ] = []
+
+            for channel_reference in rule.get("channel_references", []):
+                for ref, val in channel_reference.items():
+                    name = val["name"]
+                    component = val.get("component")
+
+                    channel_references.append(
+                        {
+                            "channel_reference": ref,
+                            "channel_identifier": channel_fqn(name, component),
+                        }
+                    )
+
+            expression = rule.get("expression", "")
+            rule_client_key = rule.get("rule_client_key", "")
+            if isinstance(expression, str):
+                rules.append(
+                    RuleConfig(
+                        name=rule["name"],
+                        description=description,
+                        expression=expression,
+                        action=action,
+                        rule_client_key=rule_client_key,
+                        channel_references=channel_references,
+                    )
+                )
+            else:
+                expression_name = cast(str, expression.get("name"))
+
+                expr = named_expressions.get(expression_name)
+
+                if expr is None:
+                    raise TelemetryConfigValidationError(
+                        f"Named expression '{expression_name}' could not be found. Make sure it was loaded in."
+                    )
+
+                sub_expressions = rule.get("sub_expressions", [])
+
+                sub_exprs: Dict[str, Any] = {}
+                for sub_expression in sub_expressions:
+                    for iden, value in sub_expression.items():
+                        sub_exprs[iden] = value
+
+                rules.append(
+                    RuleConfig(
+                        name=rule["name"],
+                        description=description,
+                        expression=expr,
+                        action=action,
+                        rule_client_key=rule_client_key,
+                        channel_references=channel_references,
+                        sub_expressions=sub_exprs,
+                    )
+                )
+
+        return cls(
+            asset_name=config_as_yaml["asset_name"],
+            ingestion_client_key=config_as_yaml["ingestion_client_key"],
+            organization_id=config_as_yaml.get("organization_id"),
+            rules=rules,
+            flows=flows,
+        )
+
+
+class TelemetryConfigValidationError(Exception):
+    """
+    When the telemetry config has invalid properties
+    """
+
+    message: str
+
+    def __init__(self, message: str):
+        super().__init__(message)

sift_py/ingestion/config/telemetry_test.py

@@ -0,0 +1,405 @@
+from pathlib import Path
+from typing import Any, Dict, cast
+
+import pytest
+import yaml
+from pytest_mock import MockerFixture, MockFixture
+
+import sift_py.ingestion.config.telemetry
+import sift_py.ingestion.config.yaml.load
+from sift_py._internal.test_util.fn import _mock_path as _mock_path_imp
+from sift_py.ingestion.channel import ChannelConfig, ChannelDataType
+from sift_py.ingestion.config.telemetry import TelemetryConfig, TelemetryConfigValidationError
+from sift_py.ingestion.config.yaml.load import (
+    _validate_yaml,
+    load_named_expression_modules,
+    read_and_validate,
+)
+from sift_py.ingestion.flow import FlowConfig
+from sift_py.ingestion.rule.config import (
+    RuleActionCreateDataReviewAnnotation,
+    RuleActionCreatePhaseAnnotation,
+    RuleActionKind,
+    RuleConfig,
+)
+
+_mock_path = _mock_path_imp(sift_py.ingestion.config.telemetry)
+
+
+def test_telemetry_config_load_from_yaml(mocker: MockFixture):
+    raw_yaml_config = cast(Dict[Any, Any], yaml.safe_load(TEST_YAML_CONFIG_STR))
+    yaml_config = _validate_yaml(raw_yaml_config)
+
+    mock_read_and_validate = mocker.patch(_mock_path(read_and_validate))
+    mock_read_and_validate.return_value = yaml_config
+
+    mock_load_named_expression_modules = mocker.patch(_mock_path(load_named_expression_modules))
+    mock_load_named_expression_modules.return_value = {
+        "log_substring_contains": "contains($1, $substr)",
+        "kinetic_energy_gt": "0.5 * $mass * $1 * $1 > $threshold",
+    }
+
+    dummy_yaml_path = Path()
+    dummy_named_expr_mod_path = Path()
+
+    telemetry_config = TelemetryConfig.try_from_yaml(dummy_yaml_path, [dummy_named_expr_mod_path])
+
+    assert telemetry_config.asset_name == "LunarVehicle426"
+    assert telemetry_config.ingestion_client_key == "lunar_vehicle_426"
+    assert len(telemetry_config.flows) == 3
+
+    flow_configs = telemetry_config.flows
+    assert flow_configs[0].name == "readings"
+    assert flow_configs[1].name == "partial_readings"
+    assert flow_configs[2].name == "logs"
+
+    readings_flow, partial_readings_flow, logs_flow = flow_configs
+    assert len(readings_flow.channels) == 4
+    assert len(partial_readings_flow.channels) == 2
+    assert len(logs_flow.channels) == 1
+
+    log_channel = logs_flow.channels[0]
+    assert log_channel.name == "log"
+    assert log_channel.description == "asset logs"
+    assert log_channel.data_type == ChannelDataType.STRING
+
+    velocity_channel, voltage_channel, vehicle_state_channel, gpio_channel = readings_flow.channels
+    assert velocity_channel.name == "velocity"
+    assert velocity_channel.data_type == ChannelDataType.DOUBLE
+    assert velocity_channel.unit == "Miles Per Hour"
+    assert velocity_channel.component == "mainmotor"
+    assert velocity_channel.description == "speed"
+
+    assert voltage_channel.name == "voltage"
+    assert voltage_channel.data_type == ChannelDataType.INT_32
+    assert voltage_channel.unit == "Volts"
+    assert voltage_channel.description == "voltage at the source"
+
+    assert vehicle_state_channel.name == "vehicle_state"
+    assert vehicle_state_channel.data_type == ChannelDataType.ENUM
+    assert vehicle_state_channel.unit == "vehicle state"
+    assert vehicle_state_channel.description == "vehicle state"
+    assert len(vehicle_state_channel.enum_types) == 3
+    assert vehicle_state_channel.enum_types[0].name == "Accelerating"
+    assert vehicle_state_channel.enum_types[0].key == 0
+    assert vehicle_state_channel.enum_types[1].name == "Decelerating"
+    assert vehicle_state_channel.enum_types[1].key == 1
+    assert vehicle_state_channel.enum_types[2].name == "Stopped"
+    assert vehicle_state_channel.enum_types[2].key == 2
+
+    assert gpio_channel.name == "gpio"
+    assert gpio_channel.data_type == ChannelDataType.BIT_FIELD
+    assert gpio_channel.unit is None
+    assert gpio_channel.description == "on/off values for pins on gpio"
+    assert len(gpio_channel.bit_field_elements) == 4
+    assert gpio_channel.bit_field_elements[0].name == "12v"
+    assert gpio_channel.bit_field_elements[0].index == 0
+    assert gpio_channel.bit_field_elements[0].bit_count == 1
+    assert gpio_channel.bit_field_elements[1].name == "charge"
+    assert gpio_channel.bit_field_elements[1].index == 1
+    assert gpio_channel.bit_field_elements[1].bit_count == 2
+    assert gpio_channel.bit_field_elements[2].name == "led"
+    assert gpio_channel.bit_field_elements[2].index == 3
+    assert gpio_channel.bit_field_elements[2].bit_count == 4
+    assert gpio_channel.bit_field_elements[3].name == "heater"
+    assert gpio_channel.bit_field_elements[3].index == 7
+    assert gpio_channel.bit_field_elements[3].bit_count == 1
+
+    assert len(telemetry_config.rules) == 4
+
+    (
+        overheating_rule,
+        speeding_rule,
+        failures_rule,
+        kinetic_energy_rule,
+    ) = telemetry_config.rules
+
+    assert overheating_rule.name == "overheating"
+    assert overheating_rule.description == "Checks for vehicle overheating"
+    assert overheating_rule.expression == '$1 == "Accelerating" && $2 > 80'
+    assert overheating_rule.action.kind() == RuleActionKind.ANNOTATION  # type: ignore
+    assert isinstance(overheating_rule.action, RuleActionCreateDataReviewAnnotation)
+
+    assert speeding_rule.name == "speeding"
+    assert speeding_rule.description == "Checks high vehicle speed"
+    assert speeding_rule.expression == "$1 > 20"
+    assert overheating_rule.action.kind() == RuleActionKind.ANNOTATION
+    assert isinstance(speeding_rule.action, RuleActionCreatePhaseAnnotation)
+
+    assert failures_rule.name == "failures"
+    assert failures_rule.description == "Checks for failure logs"
+    assert failures_rule.expression == 'contains($1, "ERROR")'
+    assert overheating_rule.action.kind() == RuleActionKind.ANNOTATION
+    assert isinstance(failures_rule.action, RuleActionCreateDataReviewAnnotation)
+
+    assert kinetic_energy_rule.name == "kinetic_energy"
+    assert kinetic_energy_rule.description == "Tracks high energy output while in motion"
+    assert kinetic_energy_rule.expression == "0.5 * 10 * $1 * $1 > 470"
+    assert overheating_rule.action.kind() == RuleActionKind.ANNOTATION
+    assert isinstance(kinetic_energy_rule.action, RuleActionCreateDataReviewAnnotation)
+
+
+def test_telemetry_config_err_if_duplicate_channels_in_flow(mocker: MockerFixture):
+    """
+    Raise an error if there are duplicate channels in a flow.
+    """
+    raw_yaml_config = cast(
+        Dict[Any, Any], yaml.safe_load(DUPLICATE_CHANNEL_IN_FLOW_TELEMETRY_CONFIG)
+    )
+    yaml_config = _validate_yaml(raw_yaml_config)
+
+    mock_read_and_validate = mocker.patch(_mock_path(read_and_validate))
+    mock_read_and_validate.return_value = yaml_config
+
+    mock_load_named_expression_modules = mocker.patch(_mock_path(load_named_expression_modules))
+    mock_load_named_expression_modules.return_value = {
+        "log_substring_contains": "contains($1, $substr)",
+        "kinetic_energy_gt": "0.5 * $mass * $1 * $1 > $threshold",
+    }
+
+    dummy_yaml_path = Path()
+    dummy_named_expr_mod_path = Path()
+
+    with pytest.raises(TelemetryConfigValidationError, match="Can't have two identical channels"):
+        _ = TelemetryConfig.try_from_yaml(dummy_yaml_path, [dummy_named_expr_mod_path])
+
+
+def test_telemetry_config_named_expression_interpolation():
+    pass
+
+
+def test_telemetry_config_validations_duplicate_rules():
+    channel = ChannelConfig(
+        name="my_channel",
+        data_type=ChannelDataType.DOUBLE,
+    )
+
+    rule_on_my_channel_a = RuleConfig(
+        name="rule_a",
+        description="",
+        expression="$1 > 10",
+        channel_references=[
+            {"channel_reference": "$1", "channel_identifier": channel.fqn()},
+        ],
+        action=RuleActionCreateDataReviewAnnotation(
+            assignee="bob@example.com",
+            tags=["barometer"],
+        ),
+    )
+
+    another_rule_on_my_channel_a = RuleConfig(
+        name="rule_a",  # same name
+        description="",
+        expression="$1 > 11",
+        channel_references=[
+            {"channel_reference": "$1", "channel_identifier": channel.fqn()},
+        ],
+        action=RuleActionCreateDataReviewAnnotation(
+            assignee="bob@example.com",
+            tags=["barometer"],
+        ),
+    )
+
+    with pytest.raises(TelemetryConfigValidationError, match="Can't have two rules"):
+        TelemetryConfig(
+            asset_name="my_asset",
+            ingestion_client_key="my_asset_key",
+            organization_id="my_organization_id",
+            flows=[
+                FlowConfig(
+                    name="my_flow",
+                    channels=[channel],
+                )
+            ],
+            rules=[rule_on_my_channel_a, another_rule_on_my_channel_a],
+        )
+
+
+def test_telemetry_config_validations_duplicate_channels():
+    channel = ChannelConfig(
+        name="my_channel",
+        data_type=ChannelDataType.DOUBLE,
+    )
+
+    with pytest.raises(TelemetryConfigValidationError, match="Can't have two identical channels"):
+        TelemetryConfig(
+            asset_name="my_asset",
+            ingestion_client_key="my_asset_key",
+            organization_id="my_organization_id",
+            flows=[
+                FlowConfig(
+                    name="my_flow",
+                    channels=[
+                        channel,
+                        channel,
+                    ],
+                )
+            ],
+        )
+
+
+def test_telemetry_config_validations_flows_with_same_name():
+    channel = ChannelConfig(
+        name="my_channel",
+        data_type=ChannelDataType.DOUBLE,
+    )
+
+    channel_b = ChannelConfig(
+        name="my_other_channel",
+        data_type=ChannelDataType.DOUBLE,
+    )
+
+    with pytest.raises(TelemetryConfigValidationError, match="Can't have two flows"):
+        TelemetryConfig(
+            asset_name="my_asset",
+            ingestion_client_key="my_asset_key",
+            organization_id="my_organization_id",
+            flows=[
+                FlowConfig(
+                    name="my_flow",
+                    channels=[channel],
+                ),
+                FlowConfig(
+                    name="my_flow",
+                    channels=[channel_b],
+                ),
+            ],
+        )
+
+
+TEST_YAML_CONFIG_STR = """
+asset_name: LunarVehicle426
+ingestion_client_key: lunar_vehicle_426
+
+channels:
+  log_channel: &log_channel
+    name: log
+    data_type: string
+    description: asset logs
+
+  velocity_channel: &velocity_channel
+    name: velocity
+    data_type: double
+    description: speed
+    unit: Miles Per Hour
+    component: mainmotor
+
+  voltage_channel: &voltage_channel
+    name: voltage
+    data_type: int32
+    description: voltage at the source
+    unit: Volts
+
+  vehicle_state_channel: &vehicle_state_channel
+    name: vehicle_state
+    data_type: enum
+    description: vehicle state
+    unit: vehicle state
+    enum_types:
+      - name: Accelerating
+        key: 0
+      - name: Decelerating
+        key: 1
+      - name: Stopped
+        key: 2
+
+  gpio_channel: &gpio_channel
+    name: gpio
+    data_type: bit_field
+    description: on/off values for pins on gpio
+    bit_field_elements:
+      - name: 12v
+        index: 0
+        bit_count: 1
+      - name: charge
+        index: 1
+        bit_count: 2
+      - name: led
+        index: 3
+        bit_count: 4
+      - name: heater
+        index: 7
+        bit_count: 1
+
+rules:
+  - name: overheating
+    description: Checks for vehicle overheating
+    expression: $1 == "Accelerating" && $2 > 80
+    channel_references:
+      - $1: *vehicle_state_channel
+      - $2: *voltage_channel
+    type: review
+
+  - name: speeding
+    description: Checks high vehicle speed
+    type: phase
+    expression: $1 > 20
+    channel_references:
+      - $1: *velocity_channel
+
+  - name: failures
+    description: Checks for failure logs
+    type: review
+    assignee: homer@example.com
+    expression:
+      name: log_substring_contains
+    channel_references:
+      - $1: *log_channel
+    sub_expressions:
+      - $substr: ERROR
+    tags:
+        - foo
+        - bar
+        - baz
+
+  - name: kinetic_energy
+    description: Tracks high energy output while in motion
+    type: review
+    assignee: homer@example.com
+    expression:
+      name: kinetic_energy_gt
+    channel_references:
+      - $1: *velocity_channel
+    sub_expressions:
+      - $mass: 10
+      - $threshold: 470
+    tags:
+        - nostromo
+
+flows:
+  - name: readings
+    channels:
+      - <<: *velocity_channel
+      - <<: *voltage_channel
+      - <<: *vehicle_state_channel
+      - <<: *gpio_channel
+
+  - name: partial_readings
+    channels:
+      - <<: *velocity_channel
+      - <<: *voltage_channel
+
+  - name: logs
+    channels:
+      - <<: *log_channel
+
+"""
+
+DUPLICATE_CHANNEL_IN_FLOW_TELEMETRY_CONFIG = """
+asset_name: LunarVehicle426
+ingestion_client_key: lunar_vehicle_426
+
+channels:
+  velocity_channel: &velocity_channel
+    name: velocity
+    data_type: double
+    description: speed
+    unit: Miles Per Hour
+    component: mainmotor
+
+flows:
+  - name: readings
+    channels:
+      - <<: *velocity_channel
+      - <<: *velocity_channel
+"""