cactus-test-definitions 1.6.0__py3-none-any.whl

cactus_test_definitions/client/test_procedures.py

@@ -0,0 +1,185 @@
+from dataclasses import dataclass
+from enum import StrEnum
+from importlib import resources
+
+import yaml
+from cactus_test_definitions.client.actions import Action
+from cactus_test_definitions.client.checks import Check
+from cactus_test_definitions.client.events import Event
+from cactus_test_definitions.csipaus import CSIPAusVersion
+from cactus_test_definitions.schema import UniqueKeyLoader
+from dataclass_wizard import LoadMeta, YAMLWizard
+
+
+class TestProcedureId(StrEnum):
+    """The set of all available test ID's
+
+    This should be kept in sync with the current set of client test procedures loaded from the procedures directory"""
+
+    __test__ = False  # Prevent pytest from picking up this class
+    ALL_01 = "ALL-01"
+    ALL_02 = "ALL-02"
+    ALL_03 = "ALL-03"
+    ALL_03_REJ = "ALL-03-REJ"
+    ALL_04 = "ALL-04"
+    ALL_05 = "ALL-05"
+    ALL_06 = "ALL-06"
+    ALL_07 = "ALL-07"
+    ALL_08 = "ALL-08"
+    ALL_09 = "ALL-09"
+    ALL_10 = "ALL-10"
+    ALL_11 = "ALL-11"
+    ALL_12 = "ALL-12"
+    ALL_13 = "ALL-13"
+    ALL_14 = "ALL-14"
+    ALL_15 = "ALL-15"
+    ALL_16 = "ALL-16"
+    ALL_17 = "ALL-17"
+    ALL_18 = "ALL-18"
+    ALL_19 = "ALL-19"
+    ALL_20 = "ALL-20"
+    ALL_21 = "ALL-21"
+    ALL_22 = "ALL-22"
+    ALL_23 = "ALL-23"
+    ALL_24 = "ALL-24"
+    ALL_25 = "ALL-25"
+    ALL_25_EXT = "ALL-25-EXT"
+    ALL_26 = "ALL-26"
+    ALL_27 = "ALL-27"
+    ALL_28 = "ALL-28"
+    ALL_29 = "ALL-29"
+    ALL_30 = "ALL-30"
+    DRA_01 = "DRA-01"
+    DRA_02 = "DRA-02"
+    DRD_01 = "DRD-01"
+    DRL_01 = "DRL-01"
+    GEN_01 = "GEN-01"
+    GEN_02 = "GEN-02"
+    GEN_03 = "GEN-03"
+    GEN_04 = "GEN-04"
+    GEN_05 = "GEN-05"
+    GEN_06 = "GEN-06"
+    GEN_07 = "GEN-07"
+    GEN_08 = "GEN-08"
+    GEN_09 = "GEN-09"
+    GEN_10 = "GEN-10"
+    GEN_11 = "GEN-11"
+    GEN_12 = "GEN-12"
+    GEN_13 = "GEN-13"
+    LOA_01 = "LOA-01"
+    LOA_02 = "LOA-02"
+    LOA_03 = "LOA-03"
+    LOA_04 = "LOA-04"
+    LOA_05 = "LOA-05"
+    LOA_06 = "LOA-06"
+    LOA_07 = "LOA-07"
+    LOA_08 = "LOA-08"
+    LOA_09 = "LOA-09"
+    LOA_10 = "LOA-10"
+    LOA_11 = "LOA-11"
+    LOA_12 = "LOA-12"
+    LOA_13 = "LOA-13"
+    MUL_01 = "MUL-01"
+    MUL_02 = "MUL-02"
+    MUL_03 = "MUL-03"
+
+    # Storage extension
+    BES_01 = "BES-01"
+    BES_02 = "BES-02"
+    BES_03 = "BES-03"
+    BES_04 = "BES-04"
+
+
+@dataclass
+class Step:
+    """A step is a part of the test procedure that waits for some form of event before running a set of actions.
+
+    It's common for a step to activate other "steps" so that the state of the active test procedure can "evolve" in
+    response to client behaviour
+
+    Instructions are out-of-band operations that need performing during the step
+    e.g. disconnect DER from grid, disable power consumption etc.
+    """
+
+    event: Event  # The event to act as a trigger
+    actions: list[Action]  # The actions to execute when the trigger is met
+    instructions: list[str] | None = None
+
+
+@dataclass
+class Preconditions:
+    """Preconditions are run during the "initialization" state that precedes the start of a test. They typically
+    allow for the setup of the test.
+
+    Checks are also included to prevent a client from starting a test before they have correctly met preconditions
+
+    Instructions are out-of-band operations that need performing at the start of the test procedure
+    e.g. attach a load etc.
+
+    If immediate_start is set to True - the "initialization" step will be progressed through immediately so that the
+    client has no opportunity to interact with the server in this state. Any actions will still be executed. Do NOT
+    utilise immediate_start with precondition checks.
+    """
+
+    init_actions: list[Action] | None = None  # To be executed as the runner starts (before anything can occur)
+    immediate_start: bool = False  # If True - a test execution will have NO "pre-start" phase.
+    actions: list[Action] | None = None  # To be executed as the test case "starts" (usually on request of client)
+    checks: list[Check] | None = None  # Will prevent move from "init" state to "started" state of a test if any fail
+    instructions: list[str] | None = None
+
+
+@dataclass
+class Criteria:
+    """Criteria represent the final pass/fail analysis run after a TestProcedure completion. They can consider both
+    the final state of the test system as well as the interactions that happened while it was running"""
+
+    checks: list[Check] | None = None  # These should be run at test procedure finalization to determine pass/fail
+
+
+@dataclass
+class TestProcedure(YAMLWizard):
+    """Top level object for collecting everything relevant to a single TestProcedure"""
+
+    __test__ = False  # Prevent pytest from picking up this class
+    description: str  # Metadata from test definitions
+    category: str  # Metadata from test definitions
+    classes: list[str]  # Metadata from test definitions
+    target_versions: list[CSIPAusVersion]  # What version(s) of csip-aus is this test targeting?
+    steps: dict[str, Step]
+    preconditions: Preconditions | None = None  # These execute during "init" and setup the test for a valid start state
+    criteria: Criteria | None = None  # How will success/failure of this procedure be determined?
+
+
+LoadMeta(raise_on_unknown_json_key=True).bind_to(TestProcedure)
+
+
+def parse_test_procedure(yaml_contents: str) -> TestProcedure:
+    """Given a YAML string - parse a TestProcedure.
+
+    This will ensure the YAML parser will use all the "strict" extensions to reduce the incidence of errors"""
+
+    return TestProcedure.from_yaml(
+        yaml_contents,
+        decoder=yaml.load,  # type: ignore
+        Loader=UniqueKeyLoader,
+    )
+
+
+def get_yaml_contents(test_procedure_id: TestProcedureId) -> str:
+    """Finds the YAML contents for the TestProcedure with the specified TestProcedureId"""
+    yaml_resource = resources.files("cactus_test_definitions.client.procedures") / f"{test_procedure_id}.yaml"
+    with resources.as_file(yaml_resource) as yaml_file:
+        with open(yaml_file, "r") as f:
+            yaml_contents = f.read()
+            return yaml_contents
+
+
+def get_test_procedure(test_procedure_id: TestProcedureId) -> TestProcedure:
+    """Gets the TestProcedure with the nominated ID by loading its definition from disk"""
+    yaml_contents = get_yaml_contents(test_procedure_id)
+    return parse_test_procedure(yaml_contents)
+
+
+def get_all_test_procedures() -> dict[TestProcedureId, TestProcedure]:
+    """Gets every TestProcedure, keyed by their TestProcedureId"""
+    return {tp_id: get_test_procedure(tp_id) for tp_id in TestProcedureId}

cactus_test_definitions/client/validate.py

@@ -0,0 +1,98 @@
+from cactus_test_definitions.client.actions import Action, validate_action_parameters
+from cactus_test_definitions.client.checks import validate_check_parameters
+from cactus_test_definitions.client.events import validate_event_parameters
+from cactus_test_definitions.client.test_procedures import (
+    TestProcedure,
+    TestProcedureId,
+)
+from cactus_test_definitions.errors import TestProcedureDefinitionError
+
+
+def validate_action(
+    procedure: TestProcedure, test_procedure_id: TestProcedureId, location: str, action: Action
+) -> None:
+    """Handles the full validation of an action's definition for a parent procedure.
+
+    procedure: The parent TestProcedure for action
+    test_procedure_id: The name of procedure (used for labelling errors)
+    location: Where in procedure can you find action? (used for labelling errors)
+    action: The action to validate
+
+    raises TestProcedureDefinitionError on failure
+    """
+    validate_action_parameters(test_procedure_id, location, action)
+
+    # Provide additional "action specific" validation
+    match action.type:
+        case "enable-steps" | "remove-steps":
+            for step_name in action.parameters["steps"]:
+                if step_name not in procedure.steps.keys():
+                    raise TestProcedureDefinitionError(
+                        f"{test_procedure_id}.{location}. Refers to unknown step '{step_name}'."
+                    )
+
+
+def validate_test_procedure_actions(test_procedure: TestProcedure, test_procedure_id: TestProcedureId) -> None:
+    """Validate actions of test procedure steps / preconditions
+
+    Ensure,
+    - action has the correct parameters
+    - if parameters refer to steps then those steps are defined for the test procedure
+    """
+
+    # Validate actions in the preconditions
+    if test_procedure.preconditions:
+        if test_procedure.preconditions.actions:
+            for action in test_procedure.preconditions.actions:
+                validate_action(test_procedure, test_procedure_id, "Precondition", action)
+
+        if test_procedure.preconditions.init_actions:
+            for action in test_procedure.preconditions.init_actions:
+                validate_action(test_procedure, test_procedure_id, "Precondition", action)
+
+    # Validate actions that exist on steps
+    for step_name, step in test_procedure.steps.items():
+        for action in step.actions:
+            validate_action(test_procedure, test_procedure_id, step_name, action)
+
+
+def validate_test_procedure_checks(test_procedure: TestProcedure, test_procedure_id: TestProcedureId) -> None:
+    """Validate checks of a test procedure
+
+    Ensure,
+    - check has the correct parameters
+    """
+
+    if test_procedure.criteria and test_procedure.criteria.checks:
+        for check in test_procedure.criteria.checks:
+            validate_check_parameters(f"{test_procedure_id}: Criteria", check)
+
+    if test_procedure.preconditions and test_procedure.preconditions.checks:
+        for check in test_procedure.preconditions.checks:
+            validate_check_parameters(f"{test_procedure_id}: Preconditions", check)
+
+    for step_name, step in test_procedure.steps.items():
+        if step.event.checks:
+            for check in step.event.checks:
+                validate_check_parameters(f"{test_procedure_id}: Step {step_name}", check)
+
+
+def validate_test_procedure_events(test_procedure: TestProcedure, test_procedure_id: TestProcedureId) -> None:
+    """Validate events of test procedure steps
+
+    Ensure,
+    - event has the correct parameters
+    """
+
+    if test_procedure.steps:
+        for step_name, step in test_procedure.steps.items():
+            validate_event_parameters(test_procedure_id, step_name, step.event)
+
+
+def validate_test_procedure(test_procedure: TestProcedure, test_procedure_id: TestProcedureId) -> None:
+    """Performs additional "high level" validation of a test procedure. (eg: ensuring all action names are valid)
+
+    raises TestProcedureDefinitionError on error"""
+    validate_test_procedure_actions(test_procedure, test_procedure_id)
+    validate_test_procedure_checks(test_procedure, test_procedure_id)
+    validate_test_procedure_events(test_procedure, test_procedure_id)

cactus_test_definitions/csipaus.py

@@ -0,0 +1,81 @@
+from enum import StrEnum
+
+
+class CSIPAusVersion(StrEnum):
+    """The various version identifiers for CSIP-Aus. Used for distinguishing what tests are compatible with what
+    released versions CSIP-Aus."""
+
+    RELEASE_1_2 = "v1.2"
+    BETA_1_3_STORAGE = "v1.3-beta/storage"
+
+
+class CSIPAusResource(StrEnum):
+    """Labels for each resource type that the server/client tests might reference. This is not designed to be
+    an exhaustive list of all SEP2 / CSIP-Aus models - only the entities that might have tests applied to them."""
+
+    DeviceCapability = "DeviceCapability"
+
+    Time = "Time"
+
+    MirrorUsagePointList = "MirrorUsagePointList"
+    MirrorUsagePoint = "MirrorUsagePoint"
+
+    EndDeviceList = "EndDeviceList"
+    EndDevice = "EndDevice"
+    ConnectionPoint = "ConnectionPoint"
+    Registration = "Registration"
+
+    FunctionSetAssignmentsList = "FunctionSetAssignmentsList"
+    FunctionSetAssignments = "FunctionSetAssignments"
+
+    DERProgramList = "DERProgramList"
+    DERProgram = "DERProgram"
+
+    DERControlList = "DERControlList"
+    DERControl = "DERControl"
+
+    DefaultDERControl = "DefaultDERControl"
+
+    DERList = "DERList"
+    DER = "DER"
+    DERCapability = "DERCapability"
+    DERSettings = "DERSettings"
+    DERStatus = "DERStatus"
+
+    SubscriptionList = "SubscriptionList"
+    Subscription = "Subscription"
+
+
+class CSIPAusReadingLocation(StrEnum):
+    Site = "Site"  # The reading is measured at the site's connection point
+    Device = "Device"  # The reading is measured at the actual device (behind the meter)
+
+
+class CSIPAusReadingType(StrEnum):
+    """A non exhaustive set of CSIPAus reading types / role flags that can be specified in tests"""
+
+    ActivePowerAverage = "ActivePowerAverage"
+    ActivePowerInstantaneous = "ActivePowerInstantaneous"
+    ActivePowerMaximum = "ActivePowerMaximum"
+    ActivePowerMinimum = "ActivePowerMinimum"
+
+    ReactivePowerAverage = "ReactivePowerAverage"
+    ReactivePowerInstantaneous = "ReactivePowerInstantaneous"
+    ReactivePowerMaximum = "ReactivePowerMaximum"
+    ReactivePowerMinimum = "ReactivePowerMinimum"
+
+    FrequencyAverage = "FrequencyAverage"
+    FrequencyInstantaneous = "FrequencyInstantaneous"
+    FrequencyMaximum = "FrequencyMaximum"
+    FrequencyMinimum = "FrequencyMinimum"
+
+    VoltageSinglePhaseAverage = "VoltageSinglePhaseAverage"
+    VoltageSinglePhaseInstantaneous = "VoltageSinglePhaseInstantaneous"
+    VoltageSinglePhaseMaximum = "VoltageSinglePhaseMaximum"
+    VoltageSinglePhaseMinimum = "VoltageSinglePhaseMinimum"
+
+
+def is_list_resource(resource: CSIPAusResource) -> bool:
+    """Returns true if the specified resource is classified as a list resource (i.e. it supports list query params) and
+    will return an entity with list attributes (eg 'all')"""
+    return resource.name.endswith("List")  # This is a really simple method but it should work

cactus_test_definitions/errors.py

@@ -0,0 +1,15 @@
+class TestProcedureDefinitionError(Exception):
+    __test__ = False  # Prevent pytest from picking up this class
+
+
+class UnresolvableVariableError(Exception):
+    """Raised whenever a NamedVariable cannot be resolved at test execution time (eg: database doesn't have the
+    requisite information)"""
+
+    pass
+
+
+class UnparseableVariableExpressionError(Exception):
+    """Raised whenever a raw string cannot parse to a NamedVariable/Expression"""
+
+    pass

cactus_test_definitions/parameters.py

@@ -0,0 +1,153 @@
+from dataclasses import dataclass
+from datetime import datetime
+from decimal import Decimal
+from enum import IntEnum, auto
+from typing import Any
+
+from cactus_test_definitions.csipaus import (
+    CSIPAusReadingLocation,
+    CSIPAusReadingType,
+    CSIPAusResource,
+)
+from cactus_test_definitions.errors import TestProcedureDefinitionError
+from cactus_test_definitions.variable_expressions import is_resolvable_variable
+
+
+class ParameterType(IntEnum):
+    """The various "basic" types that can be set in YAML for a parameter. Rudimentary type checking
+    will try to enforce these but they can't be guaranteed"""
+
+    String = auto()
+    Integer = auto()
+    Float = auto()
+    Boolean = auto()
+    DateTime = auto()  # TZ aware datetime
+    ListString = auto()  # List of strings
+    HexBinary = auto()
+    CSIPAusResource = auto()  # Member of cactus_test_definitions.csipaus.CSIPAusResource
+    ListCSIPAusResource = auto()  # List where each member is a cactus_test_definitions.csipaus.CSIPAusResource
+    CSIPAusReadingType = auto()  # Member of cactus_test_definitions.csipaus.CSIPAusReadingType
+    ListCSIPAusReadingType = auto()  # List where each member is a cactus_test_definitions.csipaus.CSIPAusReadingType
+    CSIPAusReadingLocation = auto()  # Member of cactus_test_definitions.csipaus.CSIPAusReadingLocation
+    ReadingTypeValues = auto()  # A dict of type dict[CSIPAusReadingType, list[float]], each list has the same length
+    UnsignedInteger = auto()
+
+
+@dataclass(frozen=True)
+class ParameterSchema:
+    """What parameters can be passed to a given action/check. Describes a single optional/mandatory field"""
+
+    mandatory: bool  # If this parameter required
+    expected_type: ParameterType
+
+
+def is_valid_parameter_type(expected_type: ParameterType, value: Any) -> bool:
+    """Returns true if the specified value "passes" as the expected type. Only performs rudimentary checks to try
+    and catch obvious misconfigurations"""
+    if value is None:
+        return True  # We currently allow None to pass to params. Make it a runtime concern
+
+    if is_resolvable_variable(value):
+        return True  # Too hard to validate variable expressions. Make it a runtime concern
+
+    match expected_type:
+        case ParameterType.String:
+            return isinstance(value, str)
+        case ParameterType.Integer:
+            if isinstance(value, int):
+                return True
+            else:
+                # Floats/decimals can pass through so long as they have 0 decimal places
+                try:
+                    return int(value) == value
+                except Exception:
+                    return False
+        case ParameterType.UnsignedInteger:
+            # Integer that is greater than or equal to 0
+            return is_valid_parameter_type(ParameterType.Integer, value) and value >= 0
+        case ParameterType.Float:
+            return isinstance(value, float) or isinstance(value, Decimal) or isinstance(value, int)
+        case ParameterType.Boolean:
+            return isinstance(value, bool)
+        case ParameterType.DateTime:
+            return isinstance(value, datetime)
+        case ParameterType.ListString:
+            return isinstance(value, list) and all((isinstance(e, str) for e in value))
+        case ParameterType.HexBinary:
+            try:
+                int(value, 16)
+                return True
+            except Exception:
+                return False
+        case ParameterType.CSIPAusResource:
+            try:
+                return CSIPAusResource(value) == value
+            except Exception:
+                return False
+        case ParameterType.ListCSIPAusResource:
+            return isinstance(value, list) and all(
+                (is_valid_parameter_type(ParameterType.CSIPAusResource, e) for e in value)
+            )
+        case ParameterType.CSIPAusReadingType:
+            try:
+                return CSIPAusReadingType(value) == value
+            except Exception:
+                return False
+        case ParameterType.ListCSIPAusReadingType:
+            return isinstance(value, list) and all(
+                (is_valid_parameter_type(ParameterType.CSIPAusReadingType, e) for e in value)
+            )
+        case ParameterType.CSIPAusReadingLocation:
+            try:
+                return CSIPAusReadingLocation(value) == value
+            except Exception:
+                return False
+        case ParameterType.ReadingTypeValues:
+            if not value or not isinstance(value, dict):
+                return False
+
+            last_length: int | None = None
+            for reading_type, reading_vals in value.items():
+                if (
+                    not is_valid_parameter_type(ParameterType.CSIPAusReadingType, reading_type)
+                    or not isinstance(reading_vals, list)
+                    or not all((is_valid_parameter_type(ParameterType.Float, rv) for rv in reading_vals))
+                ):
+                    return False
+
+                if last_length is None:
+                    last_length = len(reading_vals)
+                elif last_length != len(reading_vals):
+                    return False
+            return True
+
+    raise TestProcedureDefinitionError(f"Unexpected ParameterType: {ParameterType}")
+
+
+def validate_parameters(location: str, parameters: dict[str, Any], valid_schema: dict[str, ParameterSchema]) -> None:
+    """Validates parameters against valid_schema for the specified location label.
+
+    location: Label to decorate error messages (eg TestProcedureName.Step.Action)
+    parameters: The parameters dict to validate
+    valid_schema: The schema to validate parameters against. Keys will be the parameter names, value will be the schema
+
+    raises TestProcedureDefinitionError if parameters is invalid"""
+
+    # Check the supplied parameters match the schema definition
+    for param_name, param_value in parameters.items():
+        param_schema = valid_schema.get(param_name, None)
+        if param_schema is None:
+            raise TestProcedureDefinitionError(
+                f"{location} doesn't have a parameter {param_name}. Valid params are {set(valid_schema.keys())}"
+            )
+
+        # Check the type
+        if not is_valid_parameter_type(param_schema.expected_type, param_value):
+            raise TestProcedureDefinitionError(
+                f"{location} has parameter {param_name} expecting {param_schema.expected_type} but got {param_value}"
+            )
+
+    # Check all mandatory parameters are set
+    for param_name, param_schema in valid_schema.items():
+        if param_schema.mandatory and param_name not in parameters:
+            raise TestProcedureDefinitionError(f"{location} is missing mandatory parameter {param_name}")

cactus_test_definitions/schema.py

@@ -0,0 +1,22 @@
+import yaml
+
+
+class UniqueKeyLoader(yaml.SafeLoader):
+    """Originally sourced from https://gist.github.com/pypt/94d747fe5180851196eb
+    Prevents duplicate keys from overwriting eachother instead of raising a ValueError.
+
+    eg - consider the following YAML, it will parse OK but should be treated as an error:
+
+    my_class:
+        key1: abc
+        key1: def
+    """
+
+    def construct_mapping(self, node, deep=False):
+        mapping = set()
+        for key_node, _ in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in mapping:
+                raise ValueError(f"Duplicate {key!r} key found in YAML.")
+            mapping.add(key)
+        return super().construct_mapping(node, deep)