piphi-runtime-testkit-python 0.1.3__tar.gz → 0.1.4__tar.gz

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: piphi-runtime-testkit-python
-Version: 0.1.3
+Version: 0.1.4
 Summary: Pytest-first helpers for testing PiPhi runtime integrations
 Keywords: piphi,pytest,testing,runtime,iot
 Author-Email: KelvinSan <support@piphi.network>
@@ -69,6 +69,7 @@ It includes:
 - a local mock Core server that captures outbound HTTP requests
 - builder functions for runtime headers, `/config` payloads, and `/config/sync` snapshots
 - readable assertion helpers for telemetry and event delivery
+- a behavior/telemetry conformance gate that catches unusable automation conditions
 - a pytest plugin so the fixtures are available automatically
 
 It does not try to replace:
@@ -209,6 +210,27 @@ mock_core.assert_event_sent(config_id="sensor-1", event_type="device.configured"
 
 That makes the test much easier to understand at a glance.
 
+### Behavior and Telemetry Contract Gate
+
+After a representative runtime test has emitted telemetry, validate that every
+state-backed condition advertised by `behaviors.json` can read a real metric:
+
+```python
+import json
+from pathlib import Path
+
+
+def test_behavior_fields_match_runtime_telemetry(mock_core):
+    # Configure the runtime and wait for representative telemetry first.
+    behaviors = json.loads(Path("src/behaviors.json").read_text())
+    mock_core.assert_behavior_conditions_match_telemetry(behaviors)
+```
+
+The assertion checks condition fields, declared aliases, value types, and
+template conditions. Parameter-driven dynamic conditions must be named with
+`dynamic_condition_ids` and covered by a separate focused test; they are never
+silently skipped.
+
 ## Install
 
 ### Install from PyPI

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/README.md

@@ -47,6 +47,7 @@ It includes:
 - a local mock Core server that captures outbound HTTP requests
 - builder functions for runtime headers, `/config` payloads, and `/config/sync` snapshots
 - readable assertion helpers for telemetry and event delivery
+- a behavior/telemetry conformance gate that catches unusable automation conditions
 - a pytest plugin so the fixtures are available automatically
 
 It does not try to replace:
@@ -187,6 +188,27 @@ mock_core.assert_event_sent(config_id="sensor-1", event_type="device.configured"
 
 That makes the test much easier to understand at a glance.
 
+### Behavior and Telemetry Contract Gate
+
+After a representative runtime test has emitted telemetry, validate that every
+state-backed condition advertised by `behaviors.json` can read a real metric:
+
+```python
+import json
+from pathlib import Path
+
+
+def test_behavior_fields_match_runtime_telemetry(mock_core):
+    # Configure the runtime and wait for representative telemetry first.
+    behaviors = json.loads(Path("src/behaviors.json").read_text())
+    mock_core.assert_behavior_conditions_match_telemetry(behaviors)
+```
+
+The assertion checks condition fields, declared aliases, value types, and
+template conditions. Parameter-driven dynamic conditions must be named with
+`dynamic_condition_ids` and covered by a separate focused test; they are never
+silently skipped.
+
 ## Install
 
 ### Install from PyPI

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "piphi-runtime-testkit-python"
-version = "0.1.3"
+version = "0.1.4"
 description = "Pytest-first helpers for testing PiPhi runtime integrations"
 authors = [
     { name = "KelvinSan", email = "support@piphi.network" },
@@ -55,7 +55,7 @@ release-version = "python scripts/release.py"
 dev = [
     "fastapi>=0.115,<1.0",
     "httpx>=0.27,<1.0",
-    "piphi-runtime-kit-python[fastapi]==0.4.5",
+    "piphi-runtime-kit-python[fastapi]==0.7.1",
     "pytest>=8.3,<9.0",
     "pytest-asyncio>=0.24,<1.0",
     "twine>=6.2.0",

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/src/piphi_runtime_testkit_python/__init__.py

@@ -7,17 +7,25 @@ from .builders import (
     build_core_config_row,
     build_runtime_headers,
 )
+from .behavior_contract import (
+    BehaviorTelemetryIssue,
+    assert_behavior_conditions_match_telemetry,
+    find_behavior_telemetry_issues,
+)
 from .mock_core import CapturedRequest, MockCoreServer, RUNTIME_CONFIG_FETCH_PATH
 
 __all__ = [
     "CapturedRequest",
+    "BehaviorTelemetryIssue",
     "MockCoreServer",
     "RUNTIME_CONFIG_FETCH_PATH",
     "assert_entities_response",
+    "assert_behavior_conditions_match_telemetry",
     "assert_event_sent",
     "assert_telemetry_sent",
     "build_config_payload",
     "build_config_snapshot",
     "build_core_config_row",
     "build_runtime_headers",
+    "find_behavior_telemetry_issues",
 ]

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/src/piphi_runtime_testkit_python/assertions.py

@@ -79,10 +79,10 @@ def _find_request(
         if body_event_type is None and isinstance(body, dict):
             body_event_type = body.get("type")
         if device_id is not None:
-            if isinstance(body, dict) and body.get("device_id") != device_id:
+            if isinstance(body, dict) and _pick_alias(body, "device_id", "deviceId") != device_id:
                 continue
         if config_id is not None:
-            if isinstance(body, dict) and body.get("config_id") != config_id:
+            if isinstance(body, dict) and _pick_alias(body, "config_id", "configId") != config_id:
                 continue
         if event_type is not None:
             if body_event_type != event_type:
@@ -95,18 +95,23 @@ def assert_telemetry_sent(
     mock_core: MockCoreServer,
     *,
     device_id: str | None = None,
+    config_id: str | None = None,
 ) -> CapturedRequest:
     """Assert that at least one telemetry request was captured."""
 
     if not mock_core.telemetry_requests:
         raise AssertionError("Expected telemetry to be sent to mock Core, but no telemetry requests were captured.")
 
-    if device_id is None:
+    if device_id is None and config_id is None:
         return mock_core.telemetry_requests[-1]
 
-    request = _find_request(mock_core.telemetry_requests, device_id=device_id)
+    request = _find_request(mock_core.telemetry_requests, device_id=device_id, config_id=config_id)
     if request is None:
-        raise AssertionError(f"Expected telemetry for device_id={device_id!r}, but captured device_ids were {mock_core.captured_telemetry_device_ids()!r}.")
+        raise AssertionError(
+            "Expected telemetry matching filters, but none were captured. "
+            f"device_id={device_id!r} config_id={config_id!r} "
+            f"captured_device_ids={mock_core.captured_telemetry_device_ids()!r}"
+        )
     return request
 
 

piphi_runtime_testkit_python-0.1.4/src/piphi_runtime_testkit_python/behavior_contract.py

@@ -0,0 +1,268 @@
+"""Conformance checks between automation behaviors and emitted telemetry."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class BehaviorTelemetryIssue:
+    """One behavior condition that cannot be evaluated from runtime telemetry."""
+
+    device_id: str
+    condition_id: str
+    field: str
+    aliases: tuple[str, ...]
+    reason: str
+    template_id: str | None = None
+
+    def describe(self) -> str:
+        owner = (
+            f"template {self.template_id!r}"
+            if self.template_id
+            else f"device {self.device_id!r}"
+        )
+        candidates = ", ".join(repr(item) for item in (self.field, *self.aliases))
+        return (
+            f"{owner} condition {self.condition_id!r}: {self.reason}; "
+            f"expected one of [{candidates}]"
+        )
+
+
+def _as_mapping(value: Any) -> Mapping[str, Any]:
+    return value if isinstance(value, Mapping) else {}
+
+
+def _as_list(value: Any) -> list[Any]:
+    return list(value) if isinstance(value, (list, tuple)) else []
+
+
+def _metric_samples(
+    telemetry_samples: Iterable[Mapping[str, Any]] | Mapping[str, Any],
+) -> list[Mapping[str, Any]]:
+    raw_samples = (
+        [telemetry_samples]
+        if isinstance(telemetry_samples, Mapping)
+        else list(telemetry_samples)
+    )
+    samples: list[Mapping[str, Any]] = []
+    for sample in raw_samples:
+        payload = _as_mapping(sample)
+        metrics = payload.get("metrics")
+        samples.append(_as_mapping(metrics) if isinstance(metrics, Mapping) else payload)
+    return samples
+
+
+def _field_aliases(runtime: Mapping[str, Any]) -> tuple[str, ...]:
+    raw_aliases = runtime.get("fieldAliases", runtime.get("field_aliases"))
+    return tuple(
+        dict.fromkeys(
+            str(alias or "").strip()
+            for alias in _as_list(raw_aliases)
+            if str(alias or "").strip()
+        )
+    )
+
+
+def _value_matches_type(value: Any, condition_type: str) -> bool:
+    normalized_type = str(condition_type or "").strip().lower()
+    if not normalized_type:
+        return True
+    if normalized_type == "boolean":
+        return isinstance(value, bool)
+    if normalized_type == "number":
+        return isinstance(value, (int, float)) and not isinstance(value, bool)
+    if normalized_type in {"text", "enum"}:
+        return isinstance(value, str)
+    return True
+
+
+def _condition_issue(
+    condition: Mapping[str, Any],
+    *,
+    device_id: str,
+    samples: list[Mapping[str, Any]],
+    dynamic_condition_ids: set[str],
+    template_id: str | None = None,
+) -> BehaviorTelemetryIssue | None:
+    condition_id = str(condition.get("id") or condition.get("field") or "condition").strip()
+    runtime = _as_mapping(condition.get("runtime"))
+    source = str(runtime.get("source") or condition.get("source") or "state").strip().lower()
+    if source not in {"", "state"}:
+        return None
+
+    if condition_id in dynamic_condition_ids:
+        return None
+
+    dynamic_field_param = str(
+        runtime.get("fieldFromParam") or runtime.get("field_from_param") or ""
+    ).strip()
+    if dynamic_field_param:
+        return BehaviorTelemetryIssue(
+            device_id=device_id,
+            condition_id=condition_id,
+            field=dynamic_field_param,
+            aliases=(),
+            reason="dynamic condition needs an explicit test exemption",
+            template_id=template_id,
+        )
+
+    field = str(runtime.get("field") or condition.get("field") or "").strip()
+    if not field:
+        return BehaviorTelemetryIssue(
+            device_id=device_id,
+            condition_id=condition_id,
+            field="<missing>",
+            aliases=(),
+            reason="state-backed condition does not declare a runtime field",
+            template_id=template_id,
+        )
+
+    aliases = _field_aliases(runtime)
+    candidates = (field, *aliases)
+    observed = [
+        sample[candidate]
+        for sample in samples
+        for candidate in candidates
+        if candidate in sample
+    ]
+    if not observed:
+        return BehaviorTelemetryIssue(
+            device_id=device_id,
+            condition_id=condition_id,
+            field=field,
+            aliases=aliases,
+            reason="field was not emitted by any representative telemetry sample",
+            template_id=template_id,
+        )
+
+    condition_type = str(condition.get("type") or runtime.get("type") or "").strip()
+    if condition_type and not any(
+        _value_matches_type(value, condition_type) for value in observed
+    ):
+        return BehaviorTelemetryIssue(
+            device_id=device_id,
+            condition_id=condition_id,
+            field=field,
+            aliases=aliases,
+            reason=f"emitted value does not match declared type {condition_type!r}",
+            template_id=template_id,
+        )
+    return None
+
+
+def _condition_leaves(node: Any) -> Iterable[Mapping[str, Any]]:
+    current = _as_mapping(node)
+    if not current:
+        return
+    children = _as_list(current.get("children"))
+    if children:
+        for child in children:
+            yield from _condition_leaves(child)
+        return
+    yield current
+
+
+def find_behavior_telemetry_issues(
+    behaviors: Mapping[str, Any],
+    telemetry_samples: Iterable[Mapping[str, Any]] | Mapping[str, Any],
+    *,
+    dynamic_condition_ids: Iterable[str] = (),
+) -> list[BehaviorTelemetryIssue]:
+    """Return conditions that representative runtime telemetry cannot evaluate.
+
+    ``telemetry_samples`` may contain raw metric mappings or Core telemetry
+    payloads with a ``metrics`` object. Parameter-driven dynamic conditions must
+    be named explicitly in ``dynamic_condition_ids`` and tested separately.
+    """
+
+    samples = _metric_samples(telemetry_samples)
+    dynamic_ids = {
+        str(condition_id or "").strip()
+        for condition_id in dynamic_condition_ids
+        if str(condition_id or "").strip()
+    }
+    devices = [
+        _as_mapping(device)
+        for device in _as_list(behaviors.get("devices"))
+        if isinstance(device, Mapping)
+    ]
+    devices_by_id = {
+        str(device.get("id") or "").strip(): device
+        for device in devices
+        if str(device.get("id") or "").strip()
+    }
+    options_by_device = {
+        device_id: {
+            str(condition.get("id") or "").strip(): condition
+            for condition in _as_list(device.get("conditions"))
+            if isinstance(condition, Mapping)
+        }
+        for device_id, device in devices_by_id.items()
+    }
+
+    issues: list[BehaviorTelemetryIssue] = []
+    for device_id, device in devices_by_id.items():
+        for raw_condition in _as_list(device.get("conditions")):
+            condition = _as_mapping(raw_condition)
+            issue = _condition_issue(
+                condition,
+                device_id=device_id,
+                samples=samples,
+                dynamic_condition_ids=dynamic_ids,
+            )
+            if issue:
+                issues.append(issue)
+
+    for raw_template in _as_list(behaviors.get("templates")):
+        template = _as_mapping(raw_template)
+        template_id = str(template.get("id") or "template").strip()
+        device_id = str(
+            template.get("deviceKey") or template.get("device_key") or ""
+        ).strip()
+        condition_options = options_by_device.get(device_id, {})
+        config = _as_mapping(template.get("config"))
+        condition_tree = config.get("conditionTree", config.get("condition_tree"))
+        for leaf in _condition_leaves(condition_tree):
+            option_key = str(
+                _as_mapping(leaf.get("sourceRef")).get("optionKey")
+                or _as_mapping(leaf.get("source_ref")).get("option_key")
+                or leaf.get("id")
+                or ""
+            ).strip()
+            option = condition_options.get(option_key, {})
+            merged = {**option, **leaf}
+            if option.get("runtime") and not leaf.get("runtime"):
+                merged["runtime"] = option["runtime"]
+            issue = _condition_issue(
+                merged,
+                device_id=device_id,
+                samples=samples,
+                dynamic_condition_ids=dynamic_ids,
+                template_id=template_id,
+            )
+            if issue:
+                issues.append(issue)
+    return issues
+
+
+def assert_behavior_conditions_match_telemetry(
+    behaviors: Mapping[str, Any],
+    telemetry_samples: Iterable[Mapping[str, Any]] | Mapping[str, Any],
+    *,
+    dynamic_condition_ids: Iterable[str] = (),
+) -> None:
+    """Fail a contract test when advertised conditions lack runtime telemetry."""
+
+    issues = find_behavior_telemetry_issues(
+        behaviors,
+        telemetry_samples,
+        dynamic_condition_ids=dynamic_condition_ids,
+    )
+    if issues:
+        details = "\n".join(f"- {issue.describe()}" for issue in issues)
+        raise AssertionError(
+            "Behavior conditions do not match emitted telemetry:\n" + details
+        )

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/src/piphi_runtime_testkit_python/mock_core.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 import json
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -169,7 +170,7 @@ class MockCoreServer:
 
     def captured_telemetry_device_ids(self) -> list[Any]:
         return [
-            request.json_body.get("device_id")
+            request.json_body.get("device_id", request.json_body.get("deviceId"))
             for request in self.telemetry_requests
             if isinstance(request.json_body, dict)
         ]
@@ -179,10 +180,15 @@ class MockCoreServer:
         self._server.server_close()
         self._thread.join(timeout=2)
 
-    def assert_telemetry_sent(self, *, device_id: str | None = None) -> CapturedRequest:
+    def assert_telemetry_sent(
+        self,
+        *,
+        device_id: str | None = None,
+        config_id: str | None = None,
+    ) -> CapturedRequest:
         from .assertions import assert_telemetry_sent
 
-        return assert_telemetry_sent(self, device_id=device_id)
+        return assert_telemetry_sent(self, device_id=device_id, config_id=config_id)
 
     def assert_event_sent(
         self,
@@ -199,3 +205,24 @@ class MockCoreServer:
             config_id=config_id,
             event_type=event_type,
         )
+
+    def assert_behavior_conditions_match_telemetry(
+        self,
+        behaviors: Mapping[str, Any],
+        *,
+        dynamic_condition_ids: Iterable[str] = (),
+    ) -> None:
+        """Validate behaviors against telemetry captured by this mock Core."""
+
+        from .behavior_contract import assert_behavior_conditions_match_telemetry
+
+        samples = [
+            request.json_body
+            for request in self.telemetry_requests
+            if isinstance(request.json_body, dict)
+        ]
+        assert_behavior_conditions_match_telemetry(
+            behaviors,
+            samples,
+            dynamic_condition_ids=dynamic_condition_ids,
+        )

{piphi_runtime_testkit_python-0.1.3 → piphi_runtime_testkit_python-0.1.4}/tests/test_assertions.py

@@ -23,15 +23,37 @@ def make_request(path: str, json_body: dict) -> CapturedRequest:
 def test_assert_telemetry_sent_returns_matching_request():
     mock_core = MockCoreServer()
     try:
-        mock_core.telemetry_requests.append(make_request("/api/v2/integrations/telemetry", {"device_id": "plug-1"}))
+        mock_core.telemetry_requests.append(
+            make_request(
+                "/api/v2/integrations/telemetry",
+                {"device_id": "plug-1", "config_id": "config-1"},
+            )
+        )
 
-        request = assert_telemetry_sent(mock_core, device_id="plug-1")
+        request = assert_telemetry_sent(mock_core, device_id="plug-1", config_id="config-1")
 
         assert request.json_body["device_id"] == "plug-1"
     finally:
         mock_core.shutdown()
 
 
+def test_assert_telemetry_sent_matches_camel_case_payload():
+    mock_core = MockCoreServer()
+    try:
+        mock_core.telemetry_requests.append(
+            make_request(
+                "/api/v2/integrations/telemetry",
+                {"deviceId": "plug-2", "configId": "config-2"},
+            )
+        )
+
+        request = mock_core.assert_telemetry_sent(device_id="plug-2", config_id="config-2")
+
+        assert request.json_body["configId"] == "config-2"
+    finally:
+        mock_core.shutdown()
+
+
 def test_assert_event_sent_returns_matching_request():
     mock_core = MockCoreServer()
     try:

piphi_runtime_testkit_python-0.1.4/tests/test_behavior_contract.py

@@ -0,0 +1,137 @@
+import pytest
+
+from piphi_runtime_testkit_python import (
+    CapturedRequest,
+    MockCoreServer,
+    assert_behavior_conditions_match_telemetry,
+    find_behavior_telemetry_issues,
+)
+
+
+def _behaviors() -> dict:
+    return {
+        "behaviorSchemaVersion": "integration.behaviors.v2",
+        "devices": [
+            {
+                "id": "light",
+                "name": "Light",
+                "conditions": [
+                    {
+                        "id": "is_on",
+                        "label": "Power is on",
+                        "type": "boolean",
+                        "runtime": {
+                            "source": "state",
+                            "field": "is_on",
+                            "fieldAliases": ["power_state"],
+                        },
+                    },
+                    {
+                        "id": "brightness_above",
+                        "label": "Brightness is above",
+                        "type": "number",
+                        "runtime": {"source": "state", "field": "brightness"},
+                    },
+                ],
+            }
+        ],
+        "templates": [
+            {
+                "id": "bright_light",
+                "label": "Bright light",
+                "deviceKey": "light",
+                "config": {
+                    "conditionTree": {
+                        "id": "is_on",
+                        "sourceRef": {"optionKey": "is_on"},
+                        "field": "is_on",
+                        "operator": "eq",
+                        "value": True,
+                    }
+                },
+            }
+        ],
+    }
+
+
+def test_behavior_conditions_accept_fields_aliases_types_and_templates() -> None:
+    assert_behavior_conditions_match_telemetry(
+        _behaviors(),
+        {"metrics": {"power_state": True, "brightness": 42}},
+    )
+
+
+def test_behavior_conditions_report_missing_field_and_type_mismatch() -> None:
+    issues = find_behavior_telemetry_issues(
+        _behaviors(),
+        [{"is_on": "yes"}],
+    )
+
+    assert [(issue.condition_id, issue.reason) for issue in issues] == [
+        ("is_on", "emitted value does not match declared type 'boolean'"),
+        (
+            "brightness_above",
+            "field was not emitted by any representative telemetry sample",
+        ),
+        ("is_on", "emitted value does not match declared type 'boolean'"),
+    ]
+
+
+def test_behavior_conditions_fail_with_actionable_message() -> None:
+    with pytest.raises(AssertionError, match="brightness_above"):
+        assert_behavior_conditions_match_telemetry(_behaviors(), {"is_on": True})
+
+
+def test_dynamic_conditions_require_explicit_exemption() -> None:
+    behaviors = _behaviors()
+    behaviors["devices"][0]["conditions"] = [
+        {
+            "id": "feature_equals",
+            "label": "Feature equals",
+            "runtime": {
+                "source": "state",
+                "fieldFromParam": "feature_id",
+            },
+        }
+    ]
+    behaviors["templates"] = []
+
+    issues = find_behavior_telemetry_issues(behaviors, {})
+    assert issues[0].reason == "dynamic condition needs an explicit test exemption"
+
+    assert_behavior_conditions_match_telemetry(
+        behaviors,
+        {},
+        dynamic_condition_ids={"feature_equals"},
+    )
+
+
+def test_non_state_conditions_are_outside_telemetry_gate() -> None:
+    behaviors = _behaviors()
+    behaviors["devices"][0]["conditions"] = [
+        {
+            "id": "weekday",
+            "label": "It is Monday",
+            "runtime": {"source": "calendar", "field": "weekday"},
+        }
+    ]
+    behaviors["templates"] = []
+
+    assert find_behavior_telemetry_issues(behaviors, {}) == []
+
+
+def test_mock_core_validates_behaviors_against_captured_telemetry() -> None:
+    mock_core = MockCoreServer()
+    try:
+        mock_core.telemetry_requests.append(
+            CapturedRequest(
+                method="POST",
+                path="/api/v2/integrations/telemetry",
+                headers={},
+                body=b"{}",
+                json_body={"metrics": {"is_on": True, "brightness": 42}},
+            )
+        )
+        mock_core.assert_behavior_conditions_match_telemetry(_behaviors())
+    finally:
+        mock_core.shutdown()