rasa-pro 3.11.5__py3-none-any.whl → 3.12.0__py3-none-any.whl

rasa/e2e_test/e2e_config.py

@@ -1,26 +1,31 @@
 from __future__ import annotations
 
-import dataclasses
-from dataclasses import dataclass
 from pathlib import Path
-from typing import Generator, Optional, Dict, Any
+from typing import Any, Dict, Generator, Optional, Tuple
 
 import structlog
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from rasa.e2e_test.constants import (
+    DEFAULT_E2E_TESTING_MODEL,
     E2E_CONFIG_SCHEMA_FILE_PATH,
-    KEY_LLM_AS_JUDGE,
+    KEY_EXTRA_PARAMETERS,
     KEY_LLM_E2E_TEST_CONVERSION,
+    KEY_LLM_JUDGE,
 )
 from rasa.shared.constants import (
     API_BASE_CONFIG_KEY,
     DEPLOYMENT_CONFIG_KEY,
+    EMBEDDINGS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
+    MODELS_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROVIDER_CONFIG_KEY,
 )
-from rasa.shared.exceptions import RasaException
+from rasa.shared.utils.llm import (
+    combine_custom_and_default_config,
+    resolve_model_client_config,
+)
 from rasa.shared.utils.yaml import (
     parse_raw_yaml,
     read_schema_file,
@@ -32,42 +37,103 @@ structlogger = structlog.get_logger()
 CONFTEST_PATTERNS = ["conftest.yml", "conftest.yaml"]
 
 
-class InvalidLLMConfiguration(RasaException):
-    """Exception raised when the LLM configuration is invalid."""
+class BaseModelConfig(BaseModel):
+    """Base class for model configurations used by generative assertions."""
 
-    def __init__(self, error_message: str) -> None:
-        """Creates a `InvalidLLMConfiguration`."""
-        super().__init__(error_message)
+    provider: Optional[str] = None
+    model: Optional[str] = None
+    extra_parameters: Dict[str, Any] = Field(default_factory=dict)
+    model_group: Optional[str] = None
 
 
-@dataclass
-class LLMJudgeConfig:
-    """Class for storing the configuration of the LLM-As-Judge.
+class LLMJudgeConfig(BaseModel):
+    """Class for storing the configuration of the LLM-Judge.
 
-    The LLM-As-Judge is used to measure the factual accuracy
+    The LLM-Judge is used to measure the factual correctness
     (i.e., how grounded in the source documents the response is),
      or relevance of the generated response during E2E testing.
     """
 
-    api_type: str = "openai"
-    model: str = "gpt-4o-mini"
+    llm_config: BaseModelConfig
+    embeddings: Optional[BaseModelConfig] = None
 
-    @staticmethod
-    def from_dict(config_data: Dict[str, Any]) -> LLMJudgeConfig:
+    @classmethod
+    def get_default_llm_config(cls) -> Dict[str, Any]:
+        return {
+            PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+            MODEL_CONFIG_KEY: DEFAULT_E2E_TESTING_MODEL,
+        }
+
+    @classmethod
+    def from_dict(cls, config_data: Dict[str, Any]) -> LLMJudgeConfig:
         """Loads the configuration from a dictionary."""
-        llm_type = config_data.pop("api_type", "openai")
-        if llm_type != "openai":
-            raise InvalidLLMConfiguration(
-                f"Invalid LLM type '{llm_type}'. Only 'openai' is supported."
-            )
+        embeddings = config_data.pop(EMBEDDINGS_CONFIG_KEY, None)
+        llm_config = config_data.pop("llm", {})
+
+        llm_config = resolve_model_client_config(llm_config)
+        llm_config, extra_parameters = cls.extract_attributes(llm_config)
+        llm_config = combine_custom_and_default_config(
+            llm_config, cls.get_default_llm_config()
+        )
+        embeddings_config = resolve_model_client_config(embeddings)
 
-        return LLMJudgeConfig(**config_data)
+        return LLMJudgeConfig(
+            llm_config=BaseModelConfig(extra_parameters=extra_parameters, **llm_config),
+            embeddings=BaseModelConfig(**embeddings_config)
+            if embeddings_config
+            else None,
+        )
 
-    def as_dict(self) -> Dict[str, Any]:
-        return dataclasses.asdict(self)
+    @classmethod
+    def extract_attributes(
+        cls, llm_config: Dict[str, Any]
+    ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+        """Extract the expected fields from the configuration."""
+        required_config = {}
+
+        expected_fields = [
+            PROVIDER_CONFIG_KEY,
+            MODEL_CONFIG_KEY,
+        ]
+
+        if PROVIDER_CONFIG_KEY in llm_config:
+            required_config = {
+                expected_field: llm_config.pop(expected_field, None)
+                for expected_field in expected_fields
+            }
+
+        elif MODELS_CONFIG_KEY in llm_config:
+            llm_config = llm_config.pop(MODELS_CONFIG_KEY)[0]
+
+            required_config = {
+                expected_field: llm_config.pop(expected_field, None)
+                for expected_field in expected_fields
+            }
+
+        clean_config = clean_up_config(required_config)
+        return clean_config, llm_config
+
+    @property
+    def llm_config_as_dict(self) -> Dict[str, Any]:
+        return extract_config(self.llm_config)
 
-    def get_model_uri(self) -> str:
-        return f"{self.api_type}:/{self.model}"
+    @property
+    def embeddings_config_as_dict(self) -> Dict[str, Any]:
+        if self.embeddings is None:
+            return {}
+
+        return extract_config(self.embeddings)
+
+
+def clean_up_config(config_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Remove None values from the configuration."""
+    return {key: value for key, value in config_data.items() if value}
+
+
+def extract_config(config: BaseModelConfig) -> Dict[str, Any]:
+    clean_config = clean_up_config(dict(config))
+    extra_parameters = clean_config.pop(KEY_EXTRA_PARAMETERS, {})
+    return {**clean_config, **extra_parameters}
 
 
 class LLME2ETestConverterConfig(BaseModel):
@@ -99,7 +165,10 @@ class LLME2ETestConverterConfig(BaseModel):
 
     @classmethod
     def get_default_config(cls) -> Dict[str, Any]:
-        return {PROVIDER_CONFIG_KEY: OPENAI_PROVIDER, MODEL_CONFIG_KEY: "gpt-4o-mini"}
+        return {
+            PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+            MODEL_CONFIG_KEY: DEFAULT_E2E_TESTING_MODEL,
+        }
 
     @staticmethod
     def _clean_up_config(config_data: Dict[str, Any]) -> Dict[str, Any]:
@@ -160,24 +229,16 @@ def create_llm_judge_config(test_case_path: Optional[Path]) -> LLMJudgeConfig:
         structlogger.debug("e2e_config.create_llm_judge_config.no_conftest_detected")
         return LLMJudgeConfig.from_dict(config_data)
 
-    llm_judge_config_data = config_data.get(KEY_LLM_AS_JUDGE, {})
+    llm_judge_config_data = config_data.get(KEY_LLM_JUDGE, {})
     if not llm_judge_config_data:
-        structlogger.debug("e2e_config.create_llm_judge_config.no_llm_as_judge_key")
+        structlogger.debug("e2e_config.create_llm_judge_config.no_llm_judge_key")
 
     structlogger.info(
         "e2e_config.create_llm_judge_config.success",
         llm_judge_config_data=llm_judge_config_data,
     )
 
-    try:
-        return LLMJudgeConfig.from_dict(llm_judge_config_data)
-    except InvalidLLMConfiguration as e:
-        structlogger.error(
-            "e2e_config.create_llm_judge_config.invalid_llm_configuration",
-            error_message=str(e),
-            event_info="Falling back to default configuration.",
-        )
-        return LLMJudgeConfig()
+    return LLMJudgeConfig.from_dict(llm_judge_config_data)
 
 
 def create_llm_e2e_test_converter_config(

rasa/e2e_test/e2e_config_schema.yml

@@ -1,19 +1,37 @@
 mapping:
-  llm_as_judge:
+  llm_judge:
     type: map
     mapping:
-      api_type:
-        type: str
-        required: true
-        nullable: false
-      model:
-        type: str
-        required: true
-        nullable: false
+      llm:
+        type: map
+        allowempty: true
+        mapping:
+          provider:
+            type: str
+            nullable: false
+          model:
+            type: str
+            nullable: false
+          model_group:
+            type: str
+            nullable: false
+      embeddings:
+        type: map
+        allowempty: true
+        mapping:
+          provider:
+            type: str
+            nullable: false
+          model:
+            type: str
+            nullable: false
+          model_group:
+            type: str
+            nullable: false
   llm_e2e_test_conversion:
     type: map
     mapping:
-      api_type:
+      provider:
         type: str
       model:
         type: str

rasa/e2e_test/e2e_test_case.py

@@ -1,22 +1,22 @@
-from collections import OrderedDict
-from collections import defaultdict
+from collections import OrderedDict, defaultdict
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Text, Union
 
 import structlog
 
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
 from rasa.e2e_test.assertions import Assertion
 from rasa.e2e_test.constants import (
-    KEY_ASSERTIONS,
     KEY_ASSERTION_ORDER_ENABLED,
+    KEY_ASSERTIONS,
     KEY_BOT_INPUT,
     KEY_BOT_UTTERED,
     KEY_FIXTURES,
     KEY_METADATA,
-    KEY_STUB_CUSTOM_ACTIONS,
     KEY_SLOT_NOT_SET,
     KEY_SLOT_SET,
     KEY_STEPS,
+    KEY_STUB_CUSTOM_ACTIONS,
     KEY_TEST_CASE,
     KEY_TEST_CASES,
     KEY_USER_INPUT,
@@ -551,7 +551,7 @@ class Metadata:
 class TestSuite:
     """Class for representing all top level test suite keys."""
 
-    test_cases: List[TestCase]
+    test_cases: List[Union[TestCase, DialogueUnderstandingTestCase]]
     fixtures: List[Fixture]
     metadata: List[Metadata]
     stub_custom_actions: Dict[Text, StubCustomAction]

rasa/e2e_test/e2e_test_converter.py

@@ -5,8 +5,7 @@ from dataclasses import dataclass, field
 from importlib.resources import read_text
 from pathlib import Path
 from textwrap import dedent
-from typing import List, Dict, Any
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
 import pandas as pd
 import ruamel
@@ -21,8 +20,8 @@ from rasa.e2e_test.utils.validation import read_e2e_test_schema
 from rasa.exceptions import RasaException
 from rasa.shared.utils.llm import llm_factory
 from rasa.shared.utils.yaml import (
-    validate_yaml_data_using_schema_with_assertions,
     YamlValidationException,
+    validate_yaml_data_using_schema_with_assertions,
 )
 
 structlogger = structlog.get_logger()

rasa/e2e_test/e2e_test_coverage_report.py

@@ -1,17 +1,17 @@
-from typing import List, Dict, Any, Optional, Set
+from typing import Any, Dict, List, Optional, Set
 
 import pandas as pd
 import structlog
 
 from rasa.dialogue_understanding.commands import (
-    KnowledgeAnswerCommand,
-    StartFlowCommand,
-    SetSlotCommand,
-    ClarifyCommand,
-    HumanHandoffCommand,
     CancelFlowCommand,
     ChitChatAnswerCommand,
+    ClarifyCommand,
+    HumanHandoffCommand,
+    KnowledgeAnswerCommand,
+    SetSlotCommand,
     SkipQuestionCommand,
+    StartFlowCommand,
 )
 from rasa.e2e_test.e2e_test_result import TestResult
 from rasa.shared.core.flows import FlowsList

rasa/e2e_test/e2e_test_result.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, TYPE_CHECKING, Text
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text
 
 if TYPE_CHECKING:
     from rasa.e2e_test.assertions import AssertionFailure

rasa/e2e_test/e2e_test_runner.py

@@ -5,7 +5,7 @@ import difflib
 from asyncio import CancelledError
 from collections import defaultdict
 from pathlib import Path
-from typing import Any, DefaultDict, Dict, List, Optional, Text, Tuple, Union
+from typing import Any, Callable, DefaultDict, Dict, List, Optional, Text, Tuple, Union
 from urllib.parse import urlparse
 
 import requests
@@ -18,6 +18,7 @@ from rasa.core.constants import ACTIVE_FLOW_METADATA_KEY, STEP_ID_METADATA_KEY
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.persistor import StorageType
 from rasa.core.utils import AvailableEndpoints
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
 from rasa.e2e_test.constants import TEST_CASE_NAME, TEST_FILE_NAME
 from rasa.e2e_test.e2e_config import create_llm_judge_config
 from rasa.e2e_test.e2e_test_case import (
@@ -123,11 +124,11 @@ class E2ETestRunner:
             collector: Output channel.
             steps: List of steps to run.
             sender_id: The test case name with added timestamp suffix.
-            test_case_metadata: Metadata of test case.
+            test_case_metadata: Metadata of the test case.
             input_metadata: List of metadata.
 
         Returns:
-            Test turns: {turn_sequence (int) : TestStep or ActualStepOutput}.
+            Test turns: {turn_sequence (int): TestStep or ActualStepOutput}.
         """
         turns: TEST_TURNS_TYPE = {}
         event_cursor = 0
@@ -151,9 +152,23 @@ class E2ETestRunner:
             event_cursor,
         )
 
+        # This variable tracks the position of the last user step.
+        # We use it to determine from which turn onward we should check
+        # for failures in case of the fail-fast logic.
+        last_user_step_position = 0
         for position, step in enumerate(steps):
             if step.actor != "user":
                 turns[position] = step
+
+                # Check for failures after each bot step
+                try:
+                    self.fail_fast_check(
+                        test_turns=turns,
+                        last_user_step_position=last_user_step_position,
+                    )
+                except RasaException:
+                    break
+
                 continue
             elif not step.text:
                 rasa.shared.utils.io.raise_warning(
@@ -163,6 +178,8 @@ class E2ETestRunner:
                     UserWarning,
                 )
                 continue
+            # Update to the current position because we have a valid user step
+            last_user_step_position = position
 
             metadata = test_case_metadata.metadata if test_case_metadata else {}
 
@@ -202,6 +219,26 @@ class E2ETestRunner:
             )
         return turns
 
+    def fail_fast_check(
+        self, test_turns: TEST_TURNS_TYPE, last_user_step_position: int
+    ) -> None:
+        """Checks whether there are any test failures in 'test_turns'.
+
+        Args:
+            test_turns (TEST_TURNS_TYPE): The transcript of test cases and events.
+            last_user_step_position (int): The start position in the test turns.
+
+        Raises:
+            RasaException if a failure is found.
+        """
+        failures_found = self.find_test_failures(
+            test_turns=test_turns,
+            test_case=None,
+            last_user_step_position=last_user_step_position,
+        )
+        if failures_found:
+            raise RasaException("Test failure found. Aborting early.")
+
     @staticmethod
     def merge_metadata(
         sender_id: Text,
@@ -754,13 +791,15 @@ class E2ETestRunner:
     def find_test_failures(
         cls,
         test_turns: TEST_TURNS_TYPE,
-        test_case: TestCase,
+        test_case: Optional[TestCase],
+        last_user_step_position: int = 0,
     ) -> List[Tuple[TestFailure, int]]:
         """Finds the test failures in the transcript.
 
         Args:
-            test_turns: The transcript of test cases and events.
-            test_case: The test case.
+            test_turns (TEST_TURNS_TYPE): The transcript of test cases and events.
+            test_case (Optional[TestCase]): The test case.
+            last_user_step_position (int): The start position in the test turns.
 
         Returns:
             The test failures or an empty list if there is no test failure.
@@ -769,9 +808,10 @@ class E2ETestRunner:
         # with a user step
         latest_response: ActualStepOutput = test_turns[-1]  # type: ignore[assignment]
         failures = []
-        position = 0
         match = None
-        for position in range(len(test_turns) - 1):
+        for position in range(last_user_step_position, len(test_turns) - 1):
+            if position not in test_turns:
+                continue
             turn_value = test_turns[position]
             if isinstance(turn_value, ActualStepOutput):
                 latest_response = turn_value
@@ -924,19 +964,11 @@ class E2ETestRunner:
                 ).name
                 self.agent.endpoints.action.kwargs[TEST_CASE_NAME] = test_case_name
 
-            # add timestamp suffix to ensure sender_id is unique
-            sender_id = f"{test_case_name}_{datetime.datetime.now()}"
-            test_turns = await self._run_test_case(
-                sender_id, input_fixtures, input_metadata, test_case
+            sender_id = self.generate_sender_id(test_case.name)
+            _, test_result = await self._process_test_case(
+                test_case, sender_id, input_fixtures, input_metadata
             )
 
-            if not test_case.uses_assertions():
-                test_result = self.generate_test_result(test_turns, test_case)
-            else:
-                test_result = await self.run_assertions(
-                    sender_id, test_case, input_metadata
-                )
-
             results.append(test_result)
 
             coverage = kwargs.get("coverage", False)
@@ -983,6 +1015,32 @@ class E2ETestRunner:
             input_metadata,
         )
 
+    @staticmethod
+    def generate_sender_id(test_case_name: str) -> str:
+        # add timestamp suffix to ensure sender_id is unique
+        return f"{test_case_name}_{datetime.datetime.now()}"
+
+    async def _process_test_case(
+        self,
+        test_case: TestCase,
+        sender_id: str,
+        input_fixtures: List[Fixture],
+        input_metadata: Optional[List[Metadata]],
+    ) -> Tuple[TEST_TURNS_TYPE, TestResult]:
+        """Runs a single test case and returns the test turns and result."""
+        test_turns = await self._run_test_case(
+            sender_id, input_fixtures, input_metadata, test_case
+        )
+
+        if not test_case.uses_assertions():
+            test_result = self.generate_test_result(test_turns, test_case)
+        else:
+            test_result = await self.run_assertions(
+                sender_id, test_case, input_metadata
+            )
+
+        return test_turns, test_result
+
     async def run_tests_for_fine_tuning(
         self,
         input_test_cases: List[TestCase],
@@ -1008,20 +1066,12 @@ class E2ETestRunner:
 
         for i in tqdm(range(len(input_test_cases))):
             test_case = input_test_cases[i]
-            # add timestamp suffix to ensure sender_id is unique
-            sender_id = f"{test_case.name}_{datetime.datetime.now()}"
-            test_turns = await self._run_test_case(
-                sender_id, input_fixtures, input_metadata, test_case
+            sender_id = self.generate_sender_id(test_case.name)
+
+            test_turns, test_result = await self._process_test_case(
+                test_case, sender_id, input_fixtures, input_metadata
             )
 
-            # check if the e2e test is passing, only convert passing e2e tests into
-            # conversations
-            if not test_case.uses_assertions():
-                test_result = self.generate_test_result(test_turns, test_case)
-            else:
-                test_result = await self.run_assertions(
-                    sender_id, test_case, input_metadata
-                )
             if not test_result.pass_status:
                 structlogger.warning(
                     "annotation_module.skip_test_case.failing_e2e_test",
@@ -1040,12 +1090,67 @@ class E2ETestRunner:
 
         return conversations
 
+    async def run_tests_to_convert_tests_to_du_tests(
+        self,
+        input_test_cases: List[TestCase],
+        input_fixtures: List[Fixture],
+        input_metadata: Optional[List[Metadata]],
+        converting_method: Callable[
+            [TEST_TURNS_TYPE, TestCase, bool, bool],
+            Optional[DialogueUnderstandingTestCase],
+        ],
+    ) -> Tuple[
+        List[DialogueUnderstandingTestCase], List[DialogueUnderstandingTestCase]
+    ]:
+        """Runs the test cases to convert them into dialogue understanding tests.
+
+        Converts test cases into dialogue understanding test cases.
+
+        Args:
+            input_test_cases: Input test cases.
+            input_fixtures: Input fixtures.
+            input_metadata: Input metadata.
+            converting_method: The method to convert the e2e test case into a
+              dialogue understanding test case.
+
+        Returns:
+            List of ready dialogue understanding test cases and list of
+            dialogue understanding test cases to review.
+        """
+        ready_du_test_cases = []
+        to_review_du_test_cases = []
+
+        for i in tqdm(range(len(input_test_cases))):
+            test_case = input_test_cases[i]
+            sender_id = self.generate_sender_id(test_case.name)
+
+            test_turns, test_result = await self._process_test_case(
+                test_case, sender_id, input_fixtures, input_metadata
+            )
+
+            du_test_case = converting_method(
+                test_turns,
+                test_case,
+                test_case.uses_assertions(),
+                test_result.pass_status,
+            )
+
+            if du_test_case:
+                if test_result.pass_status:
+                    ready_du_test_cases.append(du_test_case)
+                else:
+                    to_review_du_test_cases.append(du_test_case)
+
+        return ready_du_test_cases, to_review_du_test_cases
+
     @staticmethod
-    def _action_server_is_reachable(endpoints: AvailableEndpoints) -> None:
+    def _action_server_is_reachable(
+        endpoints: AvailableEndpoints, module: str = "e2e_test_runner"
+    ) -> None:
         """Calls the action server health endpoint."""
         if not endpoints.action:
             structlogger.debug(
-                "e2e_test_runner._action_server_is_reachable",
+                f"{module}._action_server_is_reachable",
                 message="No action endpoint configured. Skipping the health check "
                 "of the action server.",
             )
@@ -1053,7 +1158,7 @@ class E2ETestRunner:
 
         if endpoints.action.actions_module:
             structlogger.debug(
-                "e2e_test_runner._action_server_is_reachable",
+                f"{module}._action_server_is_reachable",
                 message="Rasa server is configured to run custom actions directly. "
                 "Skipping the health check of the action server.",
             )
@@ -1061,14 +1166,14 @@ class E2ETestRunner:
 
         if not endpoints.action.url:
             structlogger.debug(
-                "e2e_test_runner._action_server_is_reachable",
+                f"{module}._action_server_is_reachable",
                 message="Action endpoint URL is not defined in the endpoint "
                 "configuration.",
             )
             return
 
         structlogger.debug(
-            "e2e_test_runner._action_server_is_reachable",
+            f"{module}._action_server_is_reachable",
             message="Detected action URL in the endpoint configuration.\n"
             f"Action Server URL: {endpoints.action.url}\n"
             "Sending a health request to the action endpoint.",
@@ -1084,7 +1189,7 @@ class E2ETestRunner:
                 "Actions server URL is defined in your endpoint configuration as "
                 f"'{endpoints.action.url}'.\n"
                 "Please make sure your action server is running and properly "
-                "configured. Since running e2e tests without a action server may "
+                "configured. Since running tests without a action server may "
                 f"lead to unpredictable results.\n{error}"
             )
 
@@ -1096,7 +1201,7 @@ class E2ETestRunner:
             )
 
         structlogger.debug(
-            "e2e_test_runner._action_server_is_reachable",
+            f"{module}._action_server_is_reachable",
             message="Action endpoint has responded successfully.\n"
             f"Response message: {response.text}\n"
             f"Response status code: {response.status_code}.",