cactus-test-definitions 1.0.0__py3-none-any.whl

cactus_test_definitions/parameters.py

@@ -0,0 +1,149 @@
+from dataclasses import dataclass
+from datetime import datetime
+from decimal import Decimal
+from enum import IntEnum, auto
+from typing import Any
+
+from cactus_test_definitions.csipaus import (
+    CSIPAusReadingLocation,
+    CSIPAusReadingType,
+    CSIPAusResource,
+)
+from cactus_test_definitions.errors import TestProcedureDefinitionError
+from cactus_test_definitions.variable_expressions import is_resolvable_variable
+
+
+class ParameterType(IntEnum):
+    """The various "basic" types that can be set in YAML for a parameter. Rudimentary type checking
+    will try to enforce these but they can't be guaranteed"""
+
+    String = auto()
+    Integer = auto()
+    Float = auto()
+    Boolean = auto()
+    DateTime = auto()  # TZ aware datetime
+    ListString = auto()  # List of strings
+    HexBinary = auto()
+    CSIPAusResource = auto()  # Member of cactus_test_definitions.csipaus.CSIPAusResource
+    ListCSIPAusResource = auto()  # List where each member is a cactus_test_definitions.csipaus.CSIPAusResource
+    CSIPAusReadingType = auto()  # Member of cactus_test_definitions.csipaus.CSIPAusReadingType
+    ListCSIPAusReadingType = auto()  # List where each member is a cactus_test_definitions.csipaus.CSIPAusReadingType
+    CSIPAusReadingLocation = auto()  # Member of cactus_test_definitions.csipaus.CSIPAusReadingLocation
+    ReadingTypeValues = auto()  # A dict of type dict[CSIPAusReadingType, list[float]], each list has the same length
+
+
+@dataclass(frozen=True)
+class ParameterSchema:
+    """What parameters can be passed to a given action/check. Describes a single optional/mandatory field"""
+
+    mandatory: bool  # If this parameter required
+    expected_type: ParameterType
+
+
+def is_valid_parameter_type(expected_type: ParameterType, value: Any) -> bool:
+    """Returns true if the specified value "passes" as the expected type. Only performs rudimentary checks to try
+    and catch obvious misconfigurations"""
+    if value is None:
+        return True  # We currently allow None to pass to params. Make it a runtime concern
+
+    if is_resolvable_variable(value):
+        return True  # Too hard to validate variable expressions. Make it a runtime concern
+
+    match expected_type:
+        case ParameterType.String:
+            return isinstance(value, str)
+        case ParameterType.Integer:
+            if isinstance(value, int):
+                return True
+            else:
+                # Floats/decimals can pass through so long as they have 0 decimal places
+                try:
+                    return int(value) == value
+                except Exception:
+                    return False
+        case ParameterType.Float:
+            return isinstance(value, float) or isinstance(value, Decimal) or isinstance(value, int)
+        case ParameterType.Boolean:
+            return isinstance(value, bool)
+        case ParameterType.DateTime:
+            return isinstance(value, datetime)
+        case ParameterType.ListString:
+            return isinstance(value, list) and all((isinstance(e, str) for e in value))
+        case ParameterType.HexBinary:
+            try:
+                int(value, 16)
+                return True
+            except Exception:
+                return False
+        case ParameterType.CSIPAusResource:
+            try:
+                return CSIPAusResource(value) == value
+            except Exception:
+                return False
+        case ParameterType.ListCSIPAusResource:
+            return isinstance(value, list) and all(
+                (is_valid_parameter_type(ParameterType.CSIPAusResource, e) for e in value)
+            )
+        case ParameterType.CSIPAusReadingType:
+            try:
+                return CSIPAusReadingType(value) == value
+            except Exception:
+                return False
+        case ParameterType.ListCSIPAusReadingType:
+            return isinstance(value, list) and all(
+                (is_valid_parameter_type(ParameterType.CSIPAusReadingType, e) for e in value)
+            )
+        case ParameterType.CSIPAusReadingLocation:
+            try:
+                return CSIPAusReadingLocation(value) == value
+            except Exception:
+                return False
+        case ParameterType.ReadingTypeValues:
+            if not value or not isinstance(value, dict):
+                return False
+
+            last_length: int | None = None
+            for reading_type, reading_vals in value.items():
+                if (
+                    not is_valid_parameter_type(ParameterType.CSIPAusReadingType, reading_type)
+                    or not isinstance(reading_vals, list)
+                    or not all((is_valid_parameter_type(ParameterType.Float, rv) for rv in reading_vals))
+                ):
+                    return False
+
+                if last_length is None:
+                    last_length = len(reading_vals)
+                elif last_length != len(reading_vals):
+                    return False
+            return True
+
+    raise TestProcedureDefinitionError(f"Unexpected ParameterType: {ParameterType}")
+
+
+def validate_parameters(location: str, parameters: dict[str, Any], valid_schema: dict[str, ParameterSchema]) -> None:
+    """Validates parameters against valid_schema for the specified location label.
+
+    location: Label to decorate error messages (eg TestProcedureName.Step.Action)
+    parameters: The parameters dict to validate
+    valid_schema: The schema to validate parameters against. Keys will be the parameter names, value will be the schema
+
+    raises TestProcedureDefinitionError if parameters is invalid"""
+
+    # Check the supplied parameters match the schema definition
+    for param_name, param_value in parameters.items():
+        param_schema = valid_schema.get(param_name, None)
+        if param_schema is None:
+            raise TestProcedureDefinitionError(
+                f"{location} doesn't have a parameter {param_name}. Valid params are {set(valid_schema.keys())}"
+            )
+
+        # Check the type
+        if not is_valid_parameter_type(param_schema.expected_type, param_value):
+            raise TestProcedureDefinitionError(
+                f"{location} has parameter {param_name} expecting {param_schema.expected_type} but got {param_value}"
+            )
+
+    # Check all mandatory parameters are set
+    for param_name, param_schema in valid_schema.items():
+        if param_schema.mandatory and param_name not in parameters:
+            raise TestProcedureDefinitionError(f"{location} is missing mandatory parameter {param_name}")

cactus_test_definitions/schema.py

@@ -0,0 +1,22 @@
+import yaml
+
+
+class UniqueKeyLoader(yaml.SafeLoader):
+    """Originally sourced from https://gist.github.com/pypt/94d747fe5180851196eb
+    Prevents duplicate keys from overwriting eachother instead of raising a ValueError.
+
+    eg - consider the following YAML, it will parse OK but should be treated as an error:
+
+    my_class:
+        key1: abc
+        key1: def
+    """
+
+    def construct_mapping(self, node, deep=False):
+        mapping = set()
+        for key_node, _ in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in mapping:
+                raise ValueError(f"Duplicate {key!r} key found in YAML.")
+            mapping.add(key)
+        return super().construct_mapping(node, deep)

cactus_test_definitions/server/README.md

@@ -0,0 +1,170 @@
+
+# Server Test Schema
+
+This is for the **SERVER** test procedures.
+
+At its most basic level, a server test is a series of actions by a virtual "client" that will probe the server and look for unexpected behaviour/responses.
+
+## Clients and Context
+
+Each test will define one or more "virtual clients" as a precondition. These can be restricted to a specific client type (eg Aggregator or Device) or left unspecified.
+
+At the beginning of each test, the client will only know it's LFDI, PEN, PIN and certificate details. Through running actions it will discover resources and populate a local "context" which represents the last seen CSIP-Aus resources. Each client's context is seperate, so if Client A performs discovery, those resources will be invisible to Client B.
+
+## Steps Schema
+
+The most basic building block of a server `TestProcedure` is a `Step`. Each `Step` will always define a single `Action` which dictates the behaviour of a virtual client (eg sending a particular request) which is then followed by a series of `Check` objects to evaluate. In order for a step to pass it must:
+
+1) Execute with any errors (eg - able to successfully make a HTTP request to the utility server)
+2) Have all checks return passed
+
+
+Steps:
+  - id: DISCOVERY
+    action: 
+      type: discovery
+      parameters:
+        resources:
+          - DeviceCapability
+          - Time
+          - MirrorUsagePointList
+          - EndDevice
+          - DER
+    checks:
+      - type: discovered
+        parameters:
+          resources:
+            - DeviceCapability
+            - Time
+            - MirrorUsagePointList
+            - EndDevice
+            - DER
+          links:
+            - ConnectionPoint
+            - Registration
+            - DERCapability
+            - DERSettings
+            - DERStatus
+      - type: end-device
+        parameters:
+          matches_client: true
+      - type: time-synced
+
+
+Step Schema:
+```
+Steps:
+  - DESCRIPTIVE_TITLE_OF_STEP: # This is used for display
+    action: # 
+      type:  # string identifier of the action type - see table below
+      parameters: # Any parameters to modify the default behaviour of the action - see table below
+    checks: # A list of Check definitions that will need to be true for this event to trigger - see section on Checks below 
+      - type:  # string identifier of the check type - see table below
+        parameters: # Any parameters to modify the default behaviour of the check - see table below
+
+    client: # The string descriptor of the "Required Client" in the preconditions that will execute this step 
+            # (Defaults to the first required client)
+    
+    use_client_context: # The string descriptor of the "Required Client" in the preconditions whose context/memory
+                        # of discovered resources will be used (for testing cross client authentication issues)
+    instructions: # List of text strings to render while this test is executing
+    repeat_until_pass: # Most steps if failed will abort the test, setting this to true will repeat this step regularly
+                       # until a pass is recorded (eg - use it to prompt a server to inject a DERControl)
+```
+
+
+### Actions
+
+These are the currently defined `Action` elements that can be included in a test.
+
+This is an example of an `Action` elements that trigger a client to requests links from the device capability URI until all nominated resources are reached.
+
+```
+action: 
+  type: discovery
+  parameters:
+  resources:
+    - Time
+    - MirrorUsagePointList
+    - EndDevice
+    - DER
+```
+
+
+| **name** | **params** | **description** |
+| -------- | ---------- | --------------- |
+| `discovery` | `resources: list[CSIPAusResource]` `next_polling_window: bool/None` | Performs a full discovery / refresh of the client's context from DeviceCapability downwards, looking to discover the specific resources. Can be delayed until the next polling window. |
+| `notifications` | `collect: bool` `disable: bool` | If `collect`, consumes subscription notifications and inserts them into the current context, if `disable` causes the subscription notification webhook to simulate an outage (return HTTP 5XX) |
+| `wait` | `duration_seconds: int` | Performs no action for the nominated period of time |
+| `refresh-resource` | `resource: CSIPAusResource` `expect_rejection: bool/None` `expect_rejection_or_empty: bool/None` | Forces a particular resource to be refreshed (using existing hrefs in context). Can be set to expect a HTTP 4XX ErrorResponse and/or an empty list resource (if appropriate). |
+| `insert-end-device` | `force_lfdi: str/None` `expect_rejection: bool/None` | Causes the client to submit a new EndDevice registration and resolves the returned Location header |
+| `upsert-connection-point` | `connectionPointId: str` `expect_rejection: bool/None` | Causes the client to submit a new ConnectionPoint with ID for the client's EndDevice |
+| `upsert-mup` | `mup_id: str` `location: CSIPAusReadingLocation` `reading_types: list[CSIPAusReadingType]` `mmr_mrids: list[str]/None` `pow10_multiplier: int/None`  `expect_rejection: bool/None` | Submits a MirrorUsagePoint with MirrorMeterReading's. Will ensure stable MRID values for the same sets of parameters (unless overridden with mmr_mrids). `mup_id` will alias this MirrorUsagePoint for future action calls. |
+| `insert-readings` | `mup_id: str` `values: ReadingTypeValues` `expect_rejection: bool/None` | Begins the submission of readings (at MUP post rate, to an earlier call to `upsert-mup` with the same `mup_id`). Will interleave transmission with subsequent steps (non blocking) if multiple sets of values are specified |
+| `upsert-der-status` | `genConnectStatus: int/None` `operationalModeStatus: int/None` `alarmStatus: int/None` `expect_rejection: bool/None` | Sends DERStatus - validates that the server persisted the values correctly |
+| `upsert-der-capability` | `type: int` `rtgMaxW: int` `modesSupported: int` `doeModesSupported: int` | Sends DERCapability - validates that the server persisted the values correctly |
+| `upsert-der-settings` | `type: int` `setMaxW: int` `setGradW: int` `modesEnabled: int` `doeModesEnabled: int` | Sends DERSettings - validates that the server persisted the values correctly |
+| `send-malformed-der-settings` | `updatedTime_missing: bool` `modesEnabled_int: bool`  | Sends a malformed DERSettings - expects a failure and that the server will NOT change anything |
+| `send-malformed-response` | `mrid_unknown: bool` `endDeviceLFDI_unknown: bool` `response_invalid: bool`  | Sends a malformed Response (using the most recent DERControl replyTo) - expects a failure response |
+| `create-subscription` | `sub_id: str` `resource: CSIPAusResource` | Sends a new Subscription - validates that the server persisted the values correctly via Location. `sub_id` will alias this subscription for future action calls. |
+| `delete-subscription` | `sub_id: str` | Sends a deletion for a previously created Subscription. |
+| `respond-der-controls` | None | Enumerates all known DERControls and sends a Response for any that require it. |
+
+
+### Checks
+
+A `Check` is a boolean test of what the client has in its current context. They are typically defined as a success/failure condition to be run at the end of a `Step`.
+
+| **name** | **params** | **description** |
+| -------- | ---------- | --------------- |
+| `discovered` | `resources: list[CSIPAusResource]` `links: list[CSIPAusResource]` | Does the client's context have the nominated resources (or the parent resource with an appropriate link). |
+| `time-sync` | None | Does the client have a TimeResponse and does it closely map to the client's local time. |
+| `end-device` | `matches_client: bool` `matches_pin` | Is there an EndDevice that matches the client's LFDI (can be negatively asserted) and does it have a specific Registration PIN |
+| `der-program` | `minimum_count: int/None` `maximum_count: int/None` `primacy: int/None` `fsa_index: int/None` | Are there enough DERProgram(s) that satisfy the filter criteria? `fsa_index` matches DERPrograms that belong to the nth (0 based) FunctionSetAssignment |
+| `der-control` | `minimum_count: int/None` `maximum_count: int/None` `latest: bool/None` `opModImpLimW: float/None` `opModExpLimW: float/None` `opModLoadLimW: float/None` `opModGenLimW: float/None` `opModEnergize: bool/None` `opModConnect: bool/None` `opModFixedW: float/None` `rampTms: int/None` `randomizeStart: int/None` `event_status: int/None` `responseRequired: int/None` `derp_primacy: int/None` | Are there enough DERProgram(s) that satisfy the filter criteria? `latest` will ONLY match the most recent DERControl. |
+| `default-der-control` | `opModExpLimW: float/None` `opModLoadLimW: float/None` `opModGenLimW: float/None` `setGradW: int/None` |  matches any DefaultDERControl with the specified values |
+| `mirror-usage-point` | `matches: bool` `location: CSIPAusReadingLocation/None` `reading_types: list[CSIPAusReadingType]/None` `mmr_mrids: list[str]/None` `post_rate_seconds: int/None` | Does a MirrorUsagePoint exist with the specified values (or not exist if `matches` is false). Only asserts specified values. |
+| `subscription` | `matches: bool` `resource: CSIPAusResource`| Does a Subscription exist for the specified resource (or not exist if `matches` is false). |
+| `poll-rate` | `resource: CSIPAusResource` `poll_rate_seconds: int` | Does the nominated resource have the specified poll rate. |
+
+
+### Parameter Variable Resolution
+
+Any `parameter` element expects a series of name/value pairs to pass to the "parent" `Action` or `Check` . For example:
+
+```
+parameters:
+    number_param: 123
+    text_param: Text Content
+    date_param: 2020-01-02 03:04:05Z
+    csip_aus_resource: EndDeviceList
+
+```
+
+But placeholder variables may also be used to reference things that aren't known until the test is underway. For example, the following would instead set `number_param` to the current setMaxW supplied by the client while `date_param` would be set to the moment in time that the `Action`, `Check` or `Event` is being evaluated.
+
+```
+parameters:
+    number_param: $setMaxW
+    text_param: Text Content
+    date_param: $now
+
+```
+
+The following are all the `NamedVariable` types currently implemented (these are distinct from the named variables
+defined in the client test procedures)
+
+| **name** | **description** |
+| -------- | --------------- |
+| `$now` | Resolves to the current moment in time (timezone aware). Returns a datetime |
+| `$setMaxW` | Resolves to the current client configuration value for `DERSetting.setMaxW` as a number. |
+
+
+Expressions are also supported.
+
+```
+parameters:
+    number_param: $(setMaxW / 2)
+    text_param: Text Content
+    date_param: $(now - '5 mins')
+```

cactus_test_definitions/server/actions.py

@@ -0,0 +1,139 @@
+from dataclasses import dataclass
+from typing import Any
+
+from cactus_test_definitions.errors import TestProcedureDefinitionError
+from cactus_test_definitions.parameters import (
+    ParameterSchema,
+    ParameterType,
+    validate_parameters,
+)
+from cactus_test_definitions.variable_expressions import (
+    parse_variable_expression_body,
+    try_extract_variable_expression,
+)
+
+
+@dataclass
+class Action:
+    type: str
+    client: str | None = None  # use the client with this id to execute this action. If None, use the 0th client
+    parameters: dict[str, Any] = None  # type: ignore # This will be forced in __post_init__
+
+    def __post_init__(self):
+        """Some parameter values might contain variable expressions (eg: a string "$now") that needs to be replaced
+        with an parsed Expression object instead."""
+        if self.parameters is None:
+            self.parameters = {}
+
+        for k, v in self.parameters.items():
+            variable_expr = try_extract_variable_expression(v)
+            if variable_expr:
+                self.parameters[k] = parse_variable_expression_body(variable_expr, k)
+
+
+# The parameter schema for each action, keyed by the action name
+ACTION_PARAMETER_SCHEMA: dict[str, dict[str, ParameterSchema]] = {
+    "discovery": {
+        "resources": ParameterSchema(True, ParameterType.ListCSIPAusResource),  # What resources to try and resolve?
+        "next_polling_window": ParameterSchema(
+            False, ParameterType.Boolean
+        ),  # If set - delay this until the upcoming polling window (eg- wait for the next whole minute)
+    },  # Performs a full discovery / refresh of the client's context from DeviceCapability downwards
+    "notifications": {
+        "collect": ParameterSchema(
+            False, ParameterType.Boolean
+        ),  # Collects latest subscription notifications into context
+        "disable": ParameterSchema(False, ParameterType.Boolean),  # Simulates HTTP 5XX outage at the endpoint
+    },
+    "wait": {
+        "duration_seconds": ParameterSchema(True, ParameterType.Integer)
+    },  # Waits (doing nothing - blocking other step actions) until the specified time period has passed
+    "comms-status": {
+        "notifications_enabled": ParameterSchema(True, ParameterType.Boolean)  # Enable/Disble notification webhook
+    },  # Enables or disables certain communications
+    "refresh-resource": {
+        "resource": ParameterSchema(True, ParameterType.CSIPAusResource),
+        "expect_rejection": ParameterSchema(False, ParameterType.Boolean),  # if set - expect 4XX and ErrorPayload
+        "expect_rejection_or_empty": ParameterSchema(
+            False, ParameterType.Boolean
+        ),  # Similar to expect_rejection but also allow en empty list (if it's a list resource)
+    },  # Force an existing resource (in the client's context) to be re-fetched via href. Updates context on success
+    "insert-end-device": {
+        "force_lfdi": ParameterSchema(False, ParameterType.String),  # Forces the use of this LFDI
+        "expect_rejection": ParameterSchema(False, ParameterType.Boolean),  # If set - expect 4XX and ErrorPayload
+    },  # Inserts an EndDevice and then validates the returned Location header
+    "upsert-connection-point": {
+        "connectionPointId": ParameterSchema(True, ParameterType.String),
+        "expect_rejection": ParameterSchema(False, ParameterType.Boolean),  # If set - expect ErrorPayload reasonCode 1
+    },
+    "upsert-mup": {
+        "mup_id": ParameterSchema(True, ParameterType.String),  # Used to alias the returned MUP ID
+        "location": ParameterSchema(True, ParameterType.CSIPAusReadingLocation),
+        "reading_types": ParameterSchema(True, ParameterType.ListCSIPAusReadingType),
+        "expect_rejection": ParameterSchema(False, ParameterType.Boolean),  # If set - expect 4XX and ErrorPayload
+        "mmr_mrids": ParameterSchema(
+            False, ParameterType.ListString
+        ),  # Must correspond 1-1 with reading_types. Used for forcing specific mrid values
+        "pow10_multiplier": ParameterSchema(
+            False, ParameterType.Integer
+        ),  # Force the use a particular pow10. Defaults to 0 otherwise
+    },  # Register a MUP with the specified values. MMR's based on hash of current client / reading types
+    "insert-readings": {
+        "mup_id": ParameterSchema(True, ParameterType.String),  # Must be previously defined with register-mup
+        "values": ParameterSchema(
+            True, ParameterType.ReadingTypeValues
+        ),  # The sequences of values to send at the MUP post rate
+        "expect_rejection": ParameterSchema(False, ParameterType.Boolean),  # If set - expect 4XX and ErrorPayload
+    },  # Sends readings - validates that the telemetry is parsed correctly by the server
+    "upsert-der-status": {
+        "genConnectStatus": ParameterSchema(False, ParameterType.Integer),
+        "operationalModeStatus": ParameterSchema(False, ParameterType.Integer),
+        "alarmStatus": ParameterSchema(False, ParameterType.Integer),
+        "expect_rejection": ParameterSchema(False, ParameterType.Boolean),  # If set - expect 4XX and ErrorPayload
+    },  # Sends DERStatus - validates that the server persisted the values correctly
+    "upsert-der-capability": {
+        "type": ParameterSchema(True, ParameterType.Integer),
+        "rtgMaxW": ParameterSchema(True, ParameterType.Integer),
+        "modesSupported": ParameterSchema(True, ParameterType.Integer),
+        "doeModesSupported": ParameterSchema(True, ParameterType.Integer),
+    },  # Sends DERCapability - validates that the server persisted the values correctly
+    "upsert-der-settings": {
+        "setMaxW": ParameterSchema(True, ParameterType.Integer),
+        "setGradW": ParameterSchema(True, ParameterType.Integer),
+        "modesEnabled": ParameterSchema(True, ParameterType.Integer),
+        "doeModesEnabled": ParameterSchema(True, ParameterType.Integer),
+    },  # Sends DERSettings - validates that the server persisted the values correctly
+    "send-malformed-der-settings": {
+        "updatedTime_missing": ParameterSchema(True, ParameterType.Boolean),  # If true - updatedTime will be stripped
+        "modesEnabled_int": ParameterSchema(True, ParameterType.Boolean),  # If true - modesEnabled will send an int
+    },  # Sends a malformed DERSettings - expects a failure and that the server will NOT change anything
+    "send-malformed-response": {
+        "mrid_unknown": ParameterSchema(True, ParameterType.Boolean),  # If true - mrid will be random
+        "endDeviceLFDI_unknown": ParameterSchema(True, ParameterType.Boolean),  # If true - endDeviceLfdi will be random
+        "response_invalid": ParameterSchema(True, ParameterType.Boolean),  # If true - response will be a reserved value
+    },  # Sends a malformed Response (using the most recent DERControl replyTo) - expects a failure response
+    "create-subscription": {
+        "sub_id": ParameterSchema(True, ParameterType.String),  # Used to alias the returned subscription ID
+        "resource": ParameterSchema(True, ParameterType.CSIPAusResource),
+    },  # Sends a new Subscription - validates that the server persisted the values correctly via Location
+    "delete-subscription": {
+        "sub_id": ParameterSchema(True, ParameterType.String),  # Must match a previously
+    },  # Sends a Subscription deletion
+    "respond-der-controls": {},  # Enumerates all known DERControls and sends a Response for any that require it
+}
+VALID_ACTION_NAMES: set[str] = set(ACTION_PARAMETER_SCHEMA.keys())
+
+
+def validate_action_parameters(procedure_name: str, step_name: int, action: Action) -> None:
+    """Validates the action parameters for the parent TestProcedure based on the  ACTION_PARAMETER_SCHEMA
+
+    raises TestProcedureDefinitionError on failure"""
+    location = f"{procedure_name}.step[{step_name}]"  # Descriptive location
+
+    parameter_schema = ACTION_PARAMETER_SCHEMA.get(action.type, None)
+    if parameter_schema is None:
+        raise TestProcedureDefinitionError(
+            f"{location} has an invalid action name '{action.type}'. Valid Names: {VALID_ACTION_NAMES}"
+        )
+
+    validate_parameters(location, action.parameters, parameter_schema)

cactus_test_definitions/server/checks.py

@@ -0,0 +1,117 @@
+from dataclasses import dataclass
+from typing import Any
+
+from cactus_test_definitions.errors import TestProcedureDefinitionError
+from cactus_test_definitions.parameters import (
+    ParameterSchema,
+    ParameterType,
+    validate_parameters,
+)
+from cactus_test_definitions.variable_expressions import (
+    parse_variable_expression_body,
+    try_extract_variable_expression,
+)
+
+
+@dataclass
+class Check:
+    """A check represents some validation logic that runs during a Test Step and provides a pass/fail result with a
+    description. It will typically inspect the state of the client based on what it has seen from the server
+
+    eg: Ensuring that the client was able to see an EndDevice registration"""
+
+    type: str
+    parameters: dict[str, Any] = None  # type: ignore # This will be forced in __post_init__
+
+    def __post_init__(self):
+        """Some parameter values might contain variable expressions (eg: a string "$now") that needs to be replaced
+        with an parsed Expression object instead."""
+        if self.parameters is None:
+            self.parameters = {}
+        for k, v in self.parameters.items():
+            variable_expr = try_extract_variable_expression(v)
+            if variable_expr:
+                self.parameters[k] = parse_variable_expression_body(variable_expr, k)
+
+
+# The parameter schema for each action, keyed by the action name
+CHECK_PARAMETER_SCHEMA: dict[str, dict[str, ParameterSchema]] = {
+    "discovered": {
+        "resources": ParameterSchema(False, ParameterType.ListCSIPAusResource),
+        "links": ParameterSchema(False, ParameterType.ListCSIPAusResource),
+    },
+    "time-synced": {},  # Passes if the current Time resource is synced with this client's date/time
+    "end-device": {
+        "matches_client": ParameterSchema(
+            True, ParameterType.Boolean
+        ),  # assert the existence / non existence of an EndDevice for the current client
+        "matches_pin": ParameterSchema(
+            False, ParameterType.Boolean
+        ),  # if set - The matches_client criteria will ALSO check the registration PIN for the EndDevice. Default False
+    },
+    "der-program": {
+        "minimum_count": ParameterSchema(False, ParameterType.Integer),  # Needs at least this many derps to pass
+        "maximum_count": ParameterSchema(False, ParameterType.Integer),  # Needs at most this many derps to pass
+        "primacy": ParameterSchema(False, ParameterType.Integer),  # Filters derps based on this primacy value
+        "fsa_index": ParameterSchema(
+            False, ParameterType.Integer
+        ),  # Filters derps that belong to the nth (0 based) FunctionSetAssignment index
+    },
+    "der-control": {
+        "minimum_count": ParameterSchema(False, ParameterType.Integer),  # Needs at least this many controls to pass
+        "maximum_count": ParameterSchema(False, ParameterType.Integer),  # Needs at most this many controls to pass
+        "latest": ParameterSchema(False, ParameterType.Boolean),  # forces filter checks against the most recent control
+        "opModImpLimW": ParameterSchema(False, ParameterType.Float),  # Filters controls based on this value
+        "opModExpLimW": ParameterSchema(False, ParameterType.Float),  # Filters controls based on this value
+        "opModLoadLimW": ParameterSchema(False, ParameterType.Float),  # Filters controls based on this value
+        "opModGenLimW": ParameterSchema(False, ParameterType.Float),  # Filters controls based on this value
+        "opModEnergize": ParameterSchema(False, ParameterType.Boolean),  # Filters controls based on this value
+        "opModConnect": ParameterSchema(False, ParameterType.Boolean),  # Filters controls based on this value
+        "opModFixedW": ParameterSchema(False, ParameterType.Float),  # Filters controls based on this value
+        "rampTms": ParameterSchema(False, ParameterType.Integer),  # Filter on this val. 0 means negative assertion
+        "randomizeStart": ParameterSchema(False, ParameterType.Integer),  # Filter on this val (in seconds)
+        "event_status": ParameterSchema(False, ParameterType.Integer),  # Filter on Event.status value
+        "responseRequired": ParameterSchema(False, ParameterType.Integer),  # Filter on responseRequired value
+        "derp_primacy": ParameterSchema(
+            False, ParameterType.Integer
+        ),  # Filter to control's belonging to a DERProgram with this primacy value
+    },  # Matches many DERControls (specified by minimum_count) against additional other filter criteria
+    "default-der-control": {
+        "opModImpLimW": ParameterSchema(False, ParameterType.Float),
+        "opModExpLimW": ParameterSchema(False, ParameterType.Float),
+        "opModGenLimW": ParameterSchema(False, ParameterType.Float),
+        "opModLoadLimW": ParameterSchema(False, ParameterType.Float),
+        "setGradW": ParameterSchema(False, ParameterType.Integer),  # Hundredths of a percent / second
+    },  # matches any DefaultDERControl with the specified values
+    "mirror-usage-point": {
+        "matches": ParameterSchema(True, ParameterType.Boolean),  # True for positive assert, False for negative assert
+        "location": ParameterSchema(False, ParameterType.CSIPAusReadingLocation),  # If not specified - match anything
+        "reading_types": ParameterSchema(False, ParameterType.ListCSIPAusReadingType),  # If not specified - match all
+        "mmr_mrids": ParameterSchema(
+            False, ParameterType.ListString
+        ),  # Must correspond 1-1 with reading_types. Used for forcing specific mrid values
+        "post_rate_seconds": ParameterSchema(False, ParameterType.Integer),  # Only asserted if specified
+    },  # True if the matches assertion finds a MirrorUsagePoint with the specified parameters (requires exact match)
+    "subscription": {
+        "matches": ParameterSchema(True, ParameterType.Boolean),  # True for positive assert, False for negative assert
+        "resource": ParameterSchema(True, ParameterType.CSIPAusResource),
+    },  # Matches the existence/nonexistence of a subscription for the specified resource
+    "poll-rate": {
+        "resource": ParameterSchema(True, ParameterType.CSIPAusResource),
+        "poll_rate_seconds": ParameterSchema(True, ParameterType.Integer),
+    },  # Asserts a specific poll rate value
+}
+VALID_CHECK_NAMES: set[str] = set(CHECK_PARAMETER_SCHEMA.keys())
+
+
+def validate_check_parameters(procedure_name: str, check: Check) -> None:
+    """Validates the check parameters for the parent TestProcedure based on the CHECK_PARAMETER_SCHEMA
+
+    raises TestProcedureDefinitionError on failure"""
+    location = f"{procedure_name} Check: {check.type}"  # Descriptive location of this action being validated
+
+    parameter_schema = CHECK_PARAMETER_SCHEMA.get(check.type, None)
+    if parameter_schema is None:
+        raise TestProcedureDefinitionError(f"{location} not a valid action name. {VALID_CHECK_NAMES}")
+
+    validate_parameters(location, check.parameters, parameter_schema)

cactus_test_definitions/server/procedures/S-ALL-01.yaml

@@ -0,0 +1,42 @@
+Description: Discovery with Out-Of-Band registration
+Category: Registration
+Classes:
+ - A
+
+TargetVersions:
+ - v1.2
+
+Preconditions:
+  required_clients:
+    - id: client
+  
+Steps:
+  - id: DISCOVERY
+    action: 
+      type: discovery
+      parameters:
+        resources:
+          - DeviceCapability
+          - Time
+          - MirrorUsagePointList
+          - EndDevice
+          - DER
+    checks:
+      - type: discovered
+        parameters:
+          resources:
+            - DeviceCapability
+            - Time
+            - MirrorUsagePointList
+            - EndDevice
+            - DER
+          links:
+            - ConnectionPoint
+            - Registration
+            - DERCapability
+            - DERSettings
+            - DERStatus
+      - type: end-device
+        parameters:
+          matches_client: true
+      - type: time-synced