rasa-pro 3.9.18__py3-none-any.whl → 3.10.16__py3-none-any.whl

rasa/e2e_test/assertions_schema.yml

@@ -0,0 +1,106 @@
+schema;assertions:
+  extensions:
+    - pykwalify_extensions.py
+  func: require_assertion_keys
+  type: "seq"
+  matching: "any"
+  sequence:
+  - type: map
+    mapping:
+       flow_started:
+         type: str
+         nullable: false
+       flow_completed:
+         type: map
+         mapping:
+           flow_id:
+             type: str
+             required: true
+           flow_step_id:
+             type: str
+       flow_cancelled:
+         type: map
+         mapping:
+           flow_id:
+             type: str
+             required: true
+           flow_step_id:
+             type: str
+       pattern_clarification_contains:
+         type: seq
+         matching: "all"
+         nullable: false
+         sequence:
+         - type: str
+           nullable: false
+       action_executed:
+         type: str
+         nullable: false
+       slot_was_set:
+         type: seq
+         matching: "all"
+         nullable: false
+         sequence:
+         - type: map
+           mapping:
+             name:
+               type: str
+               required: true
+             value:
+               type: any
+       slot_was_not_set:
+         type: seq
+         matching: "all"
+         nullable: false
+         sequence:
+         - type: map
+           mapping:
+             name:
+               type: str
+               required: true
+             value:
+               type: any
+       bot_uttered:
+         type: map
+         nullable: false
+         mapping:
+           utter_name:
+             type: str
+             nullable: false
+           buttons:
+             type: seq
+             nullable: false
+             matching: "all"
+             sequence:
+             - type: map
+               mapping:
+                 title:
+                   type: str
+                   nullable: false
+                 payload:
+                   type: str
+                   nullable: false
+           text_matches:
+             type: str
+             nullable: false
+       generative_response_is_relevant:
+         type: map
+         mapping:
+           threshold:
+             type: float
+             required: true
+           utter_name:
+             type: str
+             nullable: false
+       generative_response_is_grounded:
+         type: map
+         mapping:
+           threshold:
+             type: float
+             required: true
+           utter_name:
+             type: str
+             nullable: false
+           ground_truth:
+             type: str
+             nullable: false

rasa/e2e_test/constants.py

@@ -1,4 +1,8 @@
 SCHEMA_FILE_PATH = "e2e_test/e2e_test_schema.yml"
+E2E_CONFIG_SCHEMA_FILE_PATH = "e2e_test/e2e_config_schema.yml"
+TEST_FILE_NAME = "test_file_name"
+TEST_CASE_NAME = "test_case_name"
+STUB_CUSTOM_ACTION_NAME_SEPARATOR = "::"
 
 KEY_FIXTURES = "fixtures"
 KEY_USER_INPUT = "user"
@@ -8,4 +12,20 @@ KEY_SLOT_SET = "slot_was_set"
 KEY_SLOT_NOT_SET = "slot_was_not_set"
 KEY_STEPS = "steps"
 KEY_TEST_CASE = "test_case"
+KEY_TEST_CASES = "test_cases"
 KEY_METADATA = "metadata"
+KEY_ASSERTIONS = "assertions"
+KEY_ASSERTION_ORDER_ENABLED = "assertion_order_enabled"
+KEY_STUB_CUSTOM_ACTIONS = "stub_custom_actions"
+
+KEY_MODEL = "model"
+KEY_LLM_AS_JUDGE = "llm_as_judge"
+KEY_LLM_E2E_TEST_CONVERSION = "llm_e2e_test_conversion"
+
+DEFAULT_E2E_INPUT_TESTS_PATH = "tests/e2e_test_cases.yml"
+DEFAULT_E2E_OUTPUT_TESTS_PATH = "tests/e2e_results.yml"
+DEFAULT_COVERAGE_OUTPUT_PATH = "e2e_coverage_results"
+
+# Test status
+STATUS_PASSED = "passed"
+STATUS_FAILED = "failed"

rasa/e2e_test/e2e_config.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import dataclasses
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Generator, Optional, Dict, Any
+
+import structlog
+from pydantic import BaseModel
+
+from rasa.e2e_test.constants import (
+    E2E_CONFIG_SCHEMA_FILE_PATH,
+    KEY_LLM_AS_JUDGE,
+    KEY_LLM_E2E_TEST_CONVERSION,
+)
+from rasa.shared.constants import (
+    API_BASE_CONFIG_KEY,
+    DEPLOYMENT_CONFIG_KEY,
+    MODEL_CONFIG_KEY,
+    OPENAI_PROVIDER,
+    PROVIDER_CONFIG_KEY,
+)
+from rasa.shared.exceptions import RasaException
+from rasa.shared.utils.yaml import (
+    parse_raw_yaml,
+    read_schema_file,
+    validate_yaml_content_using_schema,
+)
+
+structlogger = structlog.get_logger()
+
+CONFTEST_PATTERNS = ["conftest.yml", "conftest.yaml"]
+
+
+class InvalidLLMConfiguration(RasaException):
+    """Exception raised when the LLM configuration is invalid."""
+
+    def __init__(self, error_message: str) -> None:
+        """Creates a `InvalidLLMConfiguration`."""
+        super().__init__(error_message)
+
+
+@dataclass
+class LLMJudgeConfig:
+    """Class for storing the configuration of the LLM-As-Judge.
+
+    The LLM-As-Judge is used to measure the factual accuracy
+    (i.e., how grounded in the source documents the response is),
+     or relevance of the generated response during E2E testing.
+    """
+
+    api_type: str = "openai"
+    model: str = "gpt-4o-mini"
+
+    @staticmethod
+    def from_dict(config_data: Dict[str, Any]) -> LLMJudgeConfig:
+        """Loads the configuration from a dictionary."""
+        llm_type = config_data.pop("api_type", "openai")
+        if llm_type != "openai":
+            raise InvalidLLMConfiguration(
+                f"Invalid LLM type '{llm_type}'. Only 'openai' is supported."
+            )
+
+        return LLMJudgeConfig(**config_data)
+
+    def as_dict(self) -> Dict[str, Any]:
+        return dataclasses.asdict(self)
+
+    def get_model_uri(self) -> str:
+        return f"{self.api_type}:/{self.model}"
+
+
+class LLME2ETestConverterConfig(BaseModel):
+    """Class for storing the LLM configuration of the E2ETestConverter.
+
+    This configuration is used to initialize the LiteLLM client.
+    """
+
+    provider: Optional[str]
+    model: Optional[str]
+    deployment: Optional[str]
+    api_base: Optional[str]
+    extra_parameters: Optional[Dict[str, Any]]
+
+    @classmethod
+    def from_dict(cls, config_data: Dict[str, Any]) -> LLME2ETestConverterConfig:
+        """Loads the configuration from a dictionary."""
+        expected_fields = [
+            PROVIDER_CONFIG_KEY,
+            API_BASE_CONFIG_KEY,
+            DEPLOYMENT_CONFIG_KEY,
+            MODEL_CONFIG_KEY,
+        ]
+        kwargs = {
+            expected_field: config_data.pop(expected_field, None)
+            for expected_field in expected_fields
+        }
+        return cls(extra_parameters=config_data, **kwargs)
+
+    @classmethod
+    def get_default_config(cls) -> Dict[str, Any]:
+        return {PROVIDER_CONFIG_KEY: OPENAI_PROVIDER, MODEL_CONFIG_KEY: "gpt-4o-mini"}
+
+    @staticmethod
+    def _clean_up_config(config_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Remove None values from the configuration."""
+        return {key: value for key, value in config_data.items() if value}
+
+    def as_dict(self) -> Dict[str, Any]:
+        return self._clean_up_config(dict(self))
+
+
+def get_conftest_path(test_case_path: Optional[Path]) -> Optional[Path]:
+    """Get the path to the conftest.yml file.
+
+    This assumes that the conftest.yml file is in the assistant project.
+    """
+    if test_case_path is None:
+        return None
+
+    while True:
+        if test_case_path.is_file():
+            test_case_path = test_case_path.parent
+
+        matches = find_conftest_path(test_case_path)
+        try:
+            match = next(matches)
+            structlogger.debug("e2e_config.get_conftest_path.found", match=match)
+            return match
+        except StopIteration:
+            pass
+
+        plausible_config_paths = [
+            test_case_path / "config.yml",
+            test_case_path / "config",
+        ]
+        for plausible_config_path in plausible_config_paths:
+            if plausible_config_path.exists():
+                # we reached the root of the assistant project
+                return None
+        # In case of an invalid path outside the assistant project,
+        # break the loop if we reach the root
+        if test_case_path == Path("."):
+            return None
+
+        test_case_path = test_case_path.parent
+
+
+def find_conftest_path(path: Path) -> Generator[Path, None, None]:
+    """Find the path to the conftest.yml file."""
+    for pattern in CONFTEST_PATTERNS:
+        for file_path in path.rglob(pattern):
+            yield file_path
+
+
+def create_llm_judge_config(test_case_path: Optional[Path]) -> LLMJudgeConfig:
+    """Create the LLM-Judge configuration from the dictionary."""
+    config_data = read_conftest_file(test_case_path)
+    if not config_data:
+        structlogger.debug("e2e_config.create_llm_judge_config.no_conftest_detected")
+        return LLMJudgeConfig.from_dict(config_data)
+
+    llm_judge_config_data = config_data.get(KEY_LLM_AS_JUDGE, {})
+    if not llm_judge_config_data:
+        structlogger.debug("e2e_config.create_llm_judge_config.no_llm_as_judge_key")
+
+    structlogger.info(
+        "e2e_config.create_llm_judge_config.success",
+        llm_judge_config_data=llm_judge_config_data,
+    )
+
+    try:
+        return LLMJudgeConfig.from_dict(llm_judge_config_data)
+    except InvalidLLMConfiguration as e:
+        structlogger.error(
+            "e2e_config.create_llm_judge_config.invalid_llm_configuration",
+            error_message=str(e),
+            event_info="Falling back to default configuration.",
+        )
+        return LLMJudgeConfig()
+
+
+def create_llm_e2e_test_converter_config(
+    config_path: Path,
+) -> LLME2ETestConverterConfig:
+    """Create the LLME2ETestConverterConfig configuration from the dictionary."""
+    config_data = read_conftest_file(config_path)
+    if not config_data:
+        structlogger.debug(
+            "e2e_config.create_llm_e2e_test_converter_config.no_conftest_detected"
+        )
+        return LLME2ETestConverterConfig.from_dict(config_data)
+
+    llm_e2e_test_converter_config_data = config_data.get(
+        KEY_LLM_E2E_TEST_CONVERSION, {}
+    )
+    if not llm_e2e_test_converter_config_data:
+        structlogger.debug(
+            "e2e_config.create_llm_e2e_test_converter_config.no_llm_e2e_test_converter_config_key"
+        )
+
+    structlogger.info(
+        "e2e_config.create_llm_e2e_test_converter_config.success",
+        llm_e2e_test_converter_config_data=llm_e2e_test_converter_config_data,
+    )
+
+    return LLME2ETestConverterConfig.from_dict(llm_e2e_test_converter_config_data)
+
+
+def read_conftest_file(test_case_path: Optional[Path]) -> Dict[str, Any]:
+    """Read the conftest.yml file."""
+    conftest_path = get_conftest_path(test_case_path)
+    if conftest_path is None:
+        return {}
+
+    e2e_config_schema = read_schema_file(E2E_CONFIG_SCHEMA_FILE_PATH)
+    config_data = parse_raw_yaml(conftest_path.read_text())
+    validate_yaml_content_using_schema(config_data, e2e_config_schema)
+
+    return config_data

rasa/e2e_test/e2e_config_schema.yml

@@ -0,0 +1,26 @@
+mapping:
+  llm_as_judge:
+    type: map
+    mapping:
+      api_type:
+        type: str
+        required: true
+        nullable: false
+      model:
+        type: str
+        required: true
+        nullable: false
+  llm_e2e_test_conversion:
+    type: map
+    mapping:
+      api_type:
+        type: str
+      model:
+        type: str
+      deployment:
+        type: str
+      api_base:
+        type: str
+      "=":
+        type: any
+

rasa/e2e_test/e2e_test_case.py

@@ -1,20 +1,27 @@
 import logging
+from collections import OrderedDict
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Text, Union
 
-from rasa.shared.core.events import BotUttered, SlotSet, UserUttered
-
+from rasa.e2e_test.assertions import Assertion
 from rasa.e2e_test.constants import (
+    KEY_ASSERTIONS,
+    KEY_ASSERTION_ORDER_ENABLED,
     KEY_BOT_INPUT,
     KEY_BOT_UTTERED,
     KEY_FIXTURES,
     KEY_METADATA,
+    KEY_STUB_CUSTOM_ACTIONS,
     KEY_SLOT_NOT_SET,
     KEY_SLOT_SET,
     KEY_STEPS,
     KEY_TEST_CASE,
+    KEY_TEST_CASES,
     KEY_USER_INPUT,
 )
+from rasa.e2e_test.stub_custom_action import StubCustomAction
+from rasa.shared.core.events import BotUttered, SlotSet, UserUttered
+from rasa.shared.exceptions import RasaException
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +54,15 @@ class Fixture:
             },
         )
 
+    def as_dict(self) -> Dict[Text, Any]:
+        """Returns the fixture as a dictionary."""
+        return {
+            self.name: [
+                {slot_name: slot_value}
+                for slot_name, slot_value in self.slots_set.items()
+            ]
+        }
+
 
 @dataclass(frozen=True)
 class TestStep:
@@ -65,6 +81,8 @@ class TestStep:
     _slot_instance: Optional[Union[Text, Dict[Text, Any]]] = None
     _underlying: Optional[Dict[Text, Any]] = None
     metadata_name: Optional[Text] = None
+    assertions: Optional[List[Assertion]] = None
+    assertion_order_enabled: bool = False
 
     @staticmethod
     def from_dict(test_step_dict: Dict[Text, Any]) -> "TestStep":
@@ -73,7 +91,7 @@ class TestStep:
         Example:
             >>> TestStep.from_dict({"user": "hello"})
             TestStep(text="hello", actor="user")
-            >>> TestStep.from_dict({"user": "hello", metadata: "some_metadata"})
+            >>> TestStep.from_dict({"user": "hello", "metadata": "some_metadata"})
             TestStep(text="hello", actor="user", metadata_name="some_metadata")
             >>> TestStep.from_dict({"bot": "hello world"})
             TestStep(text="hello world", actor="bot")
@@ -86,6 +104,15 @@ class TestStep:
         if test_step_dict.get(KEY_SLOT_NOT_SET):
             slot_instance = test_step_dict.get(KEY_SLOT_NOT_SET)
 
+        assertions = (
+            [
+                Assertion.create_typed_assertion(data)
+                for data in test_step_dict.get(KEY_ASSERTIONS, [])
+            ]
+            if KEY_ASSERTIONS in test_step_dict
+            else None
+        )
+
         return TestStep(
             text=test_step_dict.get(
                 KEY_USER_INPUT, test_step_dict.get(KEY_BOT_INPUT, "")
@@ -99,6 +126,10 @@ class TestStep:
             _slot_instance=slot_instance,
             _underlying=test_step_dict,
             metadata_name=test_step_dict.get(KEY_METADATA, ""),
+            assertions=assertions,
+            assertion_order_enabled=test_step_dict.get(
+                KEY_ASSERTION_ORDER_ENABLED, False
+            ),
         )
 
     @staticmethod
@@ -110,7 +141,7 @@ class TestStep:
             and KEY_SLOT_SET not in test_step_dict
             and KEY_SLOT_NOT_SET not in test_step_dict
         ):
-            raise ValueError(
+            raise RasaException(
                 f"Test step is missing either the {KEY_USER_INPUT}, {KEY_BOT_INPUT}, "
                 f"{KEY_SLOT_NOT_SET}, {KEY_SLOT_SET} "
                 f"or {KEY_BOT_UTTERED} key: {test_step_dict}"
@@ -120,15 +151,70 @@ class TestStep:
             test_step_dict.get(KEY_SLOT_SET) is not None
             and test_step_dict.get(KEY_SLOT_NOT_SET) is not None
         ):
-            raise ValueError(
+            raise RasaException(
                 f"Test step has both {KEY_SLOT_SET} and {KEY_SLOT_NOT_SET} keys: "
                 f"{test_step_dict}. You must only use one of the keys in a test step."
             )
 
+        if KEY_USER_INPUT not in test_step_dict and KEY_ASSERTIONS in test_step_dict:
+            raise RasaException(
+                f"Test step with assertions must only be used with the "
+                f"'{KEY_USER_INPUT}' key: {test_step_dict}"
+            )
+
+        if (
+            KEY_USER_INPUT not in test_step_dict
+            and KEY_ASSERTION_ORDER_ENABLED in test_step_dict
+        ):
+            raise RasaException(
+                f"Test step with '{KEY_ASSERTION_ORDER_ENABLED}' key must "
+                f"only be used with the '{KEY_USER_INPUT}' key: "
+                f"{test_step_dict}"
+            )
+
+        if (
+            KEY_ASSERTION_ORDER_ENABLED in test_step_dict
+            and KEY_ASSERTIONS not in test_step_dict
+        ):
+            raise RasaException(
+                f"You must specify the '{KEY_ASSERTIONS}' key in the user test step "
+                f"where you are using '{KEY_ASSERTION_ORDER_ENABLED}' key: "
+                f"{test_step_dict}"
+            )
+
     def as_dict(self) -> Dict[Text, Any]:
         """Returns the underlying dictionary of the test step."""
         return self._underlying or {}
 
+    def as_dict_yaml_format(self) -> Dict[Text, Any]:
+        """Returns the test step as a dictionary in YAML format."""
+        if not self._underlying:
+            return {}
+
+        result = self._underlying.copy()
+
+        def _handle_slots(key: str) -> None:
+            """Slots should be a list of strings or dicts."""
+            if (
+                self._underlying
+                and key in self._underlying
+                and isinstance(self._underlying[key], OrderedDict)
+            ):
+                result[key] = [
+                    {key: value} for key, value in self._underlying[key].items()
+                ]
+            elif (
+                self._underlying
+                and key in self._underlying
+                and isinstance(self._underlying[key], str)
+            ):
+                result[key] = [self._underlying[key]]
+
+        _handle_slots(KEY_SLOT_SET)
+        _handle_slots(KEY_SLOT_NOT_SET)
+
+        return result
+
     def matches_event(self, other: Union[BotUttered, SlotSet, None]) -> bool:
         """Compares the test step with BotUttered or SlotSet event.
 
@@ -313,13 +399,25 @@ class TestCase:
             name=input_test_case.get(KEY_TEST_CASE, "default"),
             steps=steps,
             file=file,
-            line=input_test_case.lc.line + 1
-            if hasattr(input_test_case, "lc")
-            else None,
+            line=(
+                input_test_case.lc.line + 1 if hasattr(input_test_case, "lc") else None
+            ),
             fixture_names=input_test_case.get(KEY_FIXTURES),
             metadata_name=input_test_case.get(KEY_METADATA),
         )
 
+    def as_dict(self) -> Dict[Text, Any]:
+        """Returns the test case as a dictionary."""
+        result = {
+            KEY_TEST_CASE: self.name,
+            KEY_STEPS: [step.as_dict_yaml_format() for step in self.steps],
+        }
+        if self.fixture_names:
+            result[KEY_FIXTURES] = self.fixture_names
+        if self.metadata_name:
+            result[KEY_METADATA] = self.metadata_name
+        return result
+
     def file_with_line(self) -> Text:
         """Returns the file name and line number of the test case."""
         if not self.file:
@@ -328,6 +426,15 @@ class TestCase:
         line = str(self.line) if self.line is not None else ""
         return f"{self.file}:{line}"
 
+    def uses_assertions(self) -> bool:
+        """Checks if the test case uses assertions."""
+        try:
+            next(step for step in self.steps if step.assertions is not None)
+        except StopIteration:
+            return False
+
+        return True
+
 
 @dataclass(frozen=True)
 class Metadata:
@@ -356,6 +463,10 @@ class Metadata:
             },
         )
 
+    def as_dict(self) -> Dict[Text, Any]:
+        """Returns the metadata as a dictionary."""
+        return {self.name: self.metadata}
+
 
 @dataclass(frozen=True)
 class TestSuite:
@@ -364,3 +475,15 @@ class TestSuite:
     test_cases: List[TestCase]
     fixtures: List[Fixture]
     metadata: List[Metadata]
+    stub_custom_actions: Dict[Text, StubCustomAction]
+
+    def as_dict(self) -> Dict[Text, Any]:
+        """Returns the test suite as a dictionary."""
+        return {
+            KEY_FIXTURES: [fixture.as_dict() for fixture in self.fixtures],
+            KEY_METADATA: [metadata.as_dict() for metadata in self.metadata],
+            KEY_STUB_CUSTOM_ACTIONS: {
+                key: value.as_dict() for key, value in self.stub_custom_actions.items()
+            },
+            KEY_TEST_CASES: [test_case.as_dict() for test_case in self.test_cases],
+        }