rasa-pro 3.12.0.dev4__py3-none-any.whl → 3.12.0.dev6__py3-none-any.whl

rasa/e2e_test/e2e_config.py

@@ -1,26 +1,31 @@
 from __future__ import annotations
 
-import dataclasses
-from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Generator, Optional
+from typing import Any, Dict, Generator, Optional, Tuple
 
 import structlog
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from rasa.e2e_test.constants import (
+    DEFAULT_E2E_TESTING_MODEL,
     E2E_CONFIG_SCHEMA_FILE_PATH,
-    KEY_LLM_AS_JUDGE,
+    KEY_EXTRA_PARAMETERS,
     KEY_LLM_E2E_TEST_CONVERSION,
+    KEY_LLM_JUDGE,
 )
 from rasa.shared.constants import (
     API_BASE_CONFIG_KEY,
     DEPLOYMENT_CONFIG_KEY,
+    EMBEDDINGS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
+    MODELS_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROVIDER_CONFIG_KEY,
 )
-from rasa.shared.exceptions import RasaException
+from rasa.shared.utils.llm import (
+    combine_custom_and_default_config,
+    resolve_model_client_config,
+)
 from rasa.shared.utils.yaml import (
     parse_raw_yaml,
     read_schema_file,
@@ -32,42 +37,103 @@ structlogger = structlog.get_logger()
 CONFTEST_PATTERNS = ["conftest.yml", "conftest.yaml"]
 
 
-class InvalidLLMConfiguration(RasaException):
-    """Exception raised when the LLM configuration is invalid."""
+class BaseModelConfig(BaseModel):
+    """Base class for model configurations used by generative assertions."""
 
-    def __init__(self, error_message: str) -> None:
-        """Creates a `InvalidLLMConfiguration`."""
-        super().__init__(error_message)
+    provider: Optional[str] = None
+    model: Optional[str] = None
+    extra_parameters: Dict[str, Any] = Field(default_factory=dict)
+    model_group: Optional[str] = None
 
 
-@dataclass
-class LLMJudgeConfig:
-    """Class for storing the configuration of the LLM-As-Judge.
+class LLMJudgeConfig(BaseModel):
+    """Class for storing the configuration of the LLM-Judge.
 
-    The LLM-As-Judge is used to measure the factual accuracy
+    The LLM-Judge is used to measure the factual correctness
     (i.e., how grounded in the source documents the response is),
      or relevance of the generated response during E2E testing.
     """
 
-    api_type: str = "openai"
-    model: str = "gpt-4o-mini"
+    llm_config: BaseModelConfig
+    embeddings: Optional[BaseModelConfig] = None
 
-    @staticmethod
-    def from_dict(config_data: Dict[str, Any]) -> LLMJudgeConfig:
+    @classmethod
+    def get_default_llm_config(cls) -> Dict[str, Any]:
+        return {
+            PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+            MODEL_CONFIG_KEY: DEFAULT_E2E_TESTING_MODEL,
+        }
+
+    @classmethod
+    def from_dict(cls, config_data: Dict[str, Any]) -> LLMJudgeConfig:
         """Loads the configuration from a dictionary."""
-        llm_type = config_data.pop("api_type", "openai")
-        if llm_type != "openai":
-            raise InvalidLLMConfiguration(
-                f"Invalid LLM type '{llm_type}'. Only 'openai' is supported."
-            )
+        embeddings = config_data.pop(EMBEDDINGS_CONFIG_KEY, None)
+        llm_config = config_data.pop("llm", {})
+
+        llm_config = resolve_model_client_config(llm_config)
+        llm_config, extra_parameters = cls.extract_attributes(llm_config)
+        llm_config = combine_custom_and_default_config(
+            llm_config, cls.get_default_llm_config()
+        )
+        embeddings_config = resolve_model_client_config(embeddings)
 
-        return LLMJudgeConfig(**config_data)
+        return LLMJudgeConfig(
+            llm_config=BaseModelConfig(extra_parameters=extra_parameters, **llm_config),
+            embeddings=BaseModelConfig(**embeddings_config)
+            if embeddings_config
+            else None,
+        )
 
-    def as_dict(self) -> Dict[str, Any]:
-        return dataclasses.asdict(self)
+    @classmethod
+    def extract_attributes(
+        cls, llm_config: Dict[str, Any]
+    ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+        """Extract the expected fields from the configuration."""
+        required_config = {}
+
+        expected_fields = [
+            PROVIDER_CONFIG_KEY,
+            MODEL_CONFIG_KEY,
+        ]
+
+        if PROVIDER_CONFIG_KEY in llm_config:
+            required_config = {
+                expected_field: llm_config.pop(expected_field, None)
+                for expected_field in expected_fields
+            }
+
+        elif MODELS_CONFIG_KEY in llm_config:
+            llm_config = llm_config.pop(MODELS_CONFIG_KEY)[0]
+
+            required_config = {
+                expected_field: llm_config.pop(expected_field, None)
+                for expected_field in expected_fields
+            }
+
+        clean_config = clean_up_config(required_config)
+        return clean_config, llm_config
+
+    @property
+    def llm_config_as_dict(self) -> Dict[str, Any]:
+        return extract_config(self.llm_config)
 
-    def get_model_uri(self) -> str:
-        return f"{self.api_type}:/{self.model}"
+    @property
+    def embeddings_config_as_dict(self) -> Dict[str, Any]:
+        if self.embeddings is None:
+            return {}
+
+        return extract_config(self.embeddings)
+
+
+def clean_up_config(config_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Remove None values from the configuration."""
+    return {key: value for key, value in config_data.items() if value}
+
+
+def extract_config(config: BaseModelConfig) -> Dict[str, Any]:
+    clean_config = clean_up_config(dict(config))
+    extra_parameters = clean_config.pop(KEY_EXTRA_PARAMETERS, {})
+    return {**clean_config, **extra_parameters}
 
 
 class LLME2ETestConverterConfig(BaseModel):
@@ -99,7 +165,10 @@ class LLME2ETestConverterConfig(BaseModel):
 
     @classmethod
     def get_default_config(cls) -> Dict[str, Any]:
-        return {PROVIDER_CONFIG_KEY: OPENAI_PROVIDER, MODEL_CONFIG_KEY: "gpt-4o-mini"}
+        return {
+            PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+            MODEL_CONFIG_KEY: DEFAULT_E2E_TESTING_MODEL,
+        }
 
     @staticmethod
     def _clean_up_config(config_data: Dict[str, Any]) -> Dict[str, Any]:
@@ -160,24 +229,16 @@ def create_llm_judge_config(test_case_path: Optional[Path]) -> LLMJudgeConfig:
         structlogger.debug("e2e_config.create_llm_judge_config.no_conftest_detected")
         return LLMJudgeConfig.from_dict(config_data)
 
-    llm_judge_config_data = config_data.get(KEY_LLM_AS_JUDGE, {})
+    llm_judge_config_data = config_data.get(KEY_LLM_JUDGE, {})
     if not llm_judge_config_data:
-        structlogger.debug("e2e_config.create_llm_judge_config.no_llm_as_judge_key")
+        structlogger.debug("e2e_config.create_llm_judge_config.no_llm_judge_key")
 
     structlogger.info(
         "e2e_config.create_llm_judge_config.success",
         llm_judge_config_data=llm_judge_config_data,
     )
 
-    try:
-        return LLMJudgeConfig.from_dict(llm_judge_config_data)
-    except InvalidLLMConfiguration as e:
-        structlogger.error(
-            "e2e_config.create_llm_judge_config.invalid_llm_configuration",
-            error_message=str(e),
-            event_info="Falling back to default configuration.",
-        )
-        return LLMJudgeConfig()
+    return LLMJudgeConfig.from_dict(llm_judge_config_data)
 
 
 def create_llm_e2e_test_converter_config(

rasa/e2e_test/e2e_config_schema.yml

@@ -1,19 +1,37 @@
 mapping:
-  llm_as_judge:
+  llm_judge:
     type: map
     mapping:
-      api_type:
-        type: str
-        required: true
-        nullable: false
-      model:
-        type: str
-        required: true
-        nullable: false
+      llm:
+        type: map
+        allowempty: true
+        mapping:
+          provider:
+            type: str
+            nullable: false
+          model:
+            type: str
+            nullable: false
+          model_group:
+            type: str
+            nullable: false
+      embeddings:
+        type: map
+        allowempty: true
+        mapping:
+          provider:
+            type: str
+            nullable: false
+          model:
+            type: str
+            nullable: false
+          model_group:
+            type: str
+            nullable: false
   llm_e2e_test_conversion:
     type: map
     mapping:
-      api_type:
+      provider:
         type: str
       model:
         type: str

rasa/e2e_test/llm_judge_prompts/answer_relevance_prompt_template.jinja2

@@ -0,0 +1,89 @@
+You are a detail-oriented judge whose task is to generate {{ num_variations }} question variations based on the provided answer.
+Each question variation should be a question that the input answer is attempting to address.
+
+## Task Steps
+
+Follow these steps to complete the task:
+
+1. Generate {{ num_variations }} question variations that the input answer is attempting to answer.
+3. Compile all the question variations  into a JSON object.
+
+### JSON Output
+
+Format your output as a JSON object with the following attribute:
+
+    - `question_variations`: a list of strings, where each string represents the generated question that the input answer is addressing.
+
+    Example:
+
+    ```json
+    {
+        "question_variations": [
+             "Does Rasa Pro support building an AI assistant with LLMs?",
+             "In which version of Rasa Pro was support for LLM usage released?",
+        ]
+    }
+    ```
+
+### Task Requirements
+
+- Ensure that each question variation is relevant to the input answer.
+
+### Examples
+
+These are a few examples of how to generate question variations and identify non-committed answers:
+
+#### Example 1
+    - **Input Answer**:
+        ```
+        You can build a CALM assistant with Rasa Pro by defining your own business logic flows.
+        I could not identify the exact version of Rasa Pro which released CALM.
+        ```
+    - **Output**:
+        ```json
+        {
+            "question_variations": [
+                "How can you build a CALM assistant with Rasa Pro?",
+                "What is the version of Rasa Pro that released CALM?",
+                "What type of assistant can you build with Rasa Pro?",
+            ]
+        }
+        ```
+
+#### Example 2:
+    - **Input Answer**:
+        ```
+        You can integrate external knowledge sources into your Rasa Pro assistant.
+        ```
+    - **Output**:
+        ```json
+        {
+            "question_variations": [
+                "Can you integrate external knowledge sources into your Rasa Pro assistant?",
+            ]
+        }
+        ```
+
+
+#### Example 3:
+    - **Input Answer**:
+        ```
+        Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI, in version 3.7.0.
+        ```
+    - **Output**:
+        ```json
+        {
+            "question_variations": [
+                "What is CALM in Rasa Pro?",
+                "In which Rasa Pro version was CALM released?",
+                "What is the LLM-native approach in Rasa Pro?",
+            ]
+        }
+        ```
+
+
+## Task Inputs
+
+- **Input Answer**: {{ user_message }}
+
+Your output:

rasa/e2e_test/llm_judge_prompts/groundedness_prompt_template.jinja2

@@ -0,0 +1,165 @@
+You are a detail-oriented impartial judge whose task is to evaluate the correctness of a given answer based on the provided ground truth.
+
+## Task Steps
+
+Follow these steps to complete the task:
+
+1. Given the input answer, break down the answer into one or more sentences.
+2. Extract from each sentence one or more fully understandable and distinct statements while also ensuring no pronouns are used in each statement.
+3. For each statement, determine whether the statement is supported by the provided ground truth. If the statement is supported,
+mark it with the score `1`. If the statement is not supported, mark it with the score `0`.
+4. Provide a justification for the score you assigned. Each justification must reference the relevant part of the ground truth.
+5. Compile all the statements, scores, and justifications into a JSON object.
+
+### JSON Output
+
+Format your output as a JSON object with the following attribute:
+
+    - `statements`: a list of dictionaries, where each dictionary contains the following keys:
+        - `statement`: the extracted statement from the input answer
+        - `score`: the score assigned to the statement (0 or 1)
+        - `justification`: a brief and distinct explanation justifying the score assigned to the statement
+
+    Example:
+
+    ```json
+    {
+        "statements": [
+            {
+                "statement": "You can run Rasa Pro using Python 3.11 starting from Rasa Pro version 3.11.",
+                "score": 1,
+                "justification": "According to the ground truth source, support for Python 3.11 was added in Rasa Pro version 3.11"
+            },
+            {
+                "statement": "Rasa Pro does not support LLM usage in any version.",
+                "score": 0,
+                "justification": "According to the ground truth source, Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI, in version 3.7.0"
+            }
+        ]
+    }
+    ```
+
+### Task Requirements
+
+- If the answer contains multiple statements, ensure that each statement is evaluated independently.
+- If the answer contains a statement that is not verifiable by the ground truth, mark the statement as unsupported with the score `0`.
+- If the answer contains a statement that is verifiable by the ground truth, mark the statement as supported with the score `1`.
+- Provide a brief justification for each score assigned to a statement.
+
+### Examples
+
+These are a few examples of how to evaluate the correctness of the answer based on the ground truth:
+
+#### Example 1
+    - **Input Answer**:
+        ```
+        You can build a CALM assistant with Rasa Pro by defining your own business logic flows.
+        In addition, CALM leverages customizable default flows designed to handle various conversational repair scenarios.
+        CALM contains a built-in LLM approach designed to generate predefined commands that
+        reflect the user's intentions to start and stop flows, fill slots and more.
+        ```
+    - **Ground Truth**:
+        ```
+        Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI, in version 3.7.0.
+        The CALM approach has three key elements: Business Logic, Dialogue Understanding, and Automatic Conversation Repair.
+        Business logic is implemented as a set of flows. A flow describes a business process that your AI assistant can handle.
+        Dialogue understanding is designed to interpret what end users are communicating to your assistant.
+        This process involves generating commands that reflect the user's intentions, by starting and stopping flows, filling slots and more.
+        Automatic conversation repair handles all the ways conversations can go "off script". This is implemented as a set of default flows open for customization.
+        ```
+    - **Output**:
+        ```json
+        {
+            "statements": [
+                {
+                    "statement": "You can build a CALM assistant with Rasa Pro.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI"
+                },
+                {
+                    "statement": "You can define your own business logic as flows in a CALM assistant with Rasa Pro.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that business logic is implemented as a set of flows in CALM"
+                },
+                {
+                    "statement": "Conversation repair scenarios are handled by customizable default flows in CALM.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that automatic conversation repair is implemented as a set of default flows open for customization"
+                },
+                {
+                    "statement": "CALM contains a built-in LLM approach.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that CALM leverages an LLM-native approach"
+                },
+                {
+                    "statement": "The LLM approach in CALM generates predefined commands reflecting user intentions.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that dialogue understanding involves generating commands reflecting user intentions"
+                },
+                {
+                    "statement": "The LLM approach in CALM is designed to start and stop flows, fill slots, and more.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that dialogue understanding involves generating commands to start and stop flows, fill slots, and more"
+                }
+            ]
+        }
+        ```
+
+#### Example 2:
+    - **Input Answer**:
+        ```
+        You cannot integrate external knowledge into your Rasa Pro assistant.
+        ```
+    - **Ground Truth**:
+        ```
+        The Enterprise Search Policy is part of Rasa's new Conversational AI with Language Models (CALM) approach and available starting with version 3.7.0.
+        The Enterprise Search Policy uses an LLM to search external knowledge base documents in order to deliver a relevant, context-aware response from the data.
+        ```
+    - **Output**:
+        ```json
+        {
+            "statements": [
+                {
+                    "statement": "Rasa Pro does not support integrating external knowledge.",
+                    "score": 0,
+                    "justification": "The provided statement is incorrect, because the ground truth confirms that the Enterprise Search Policy in Rasa Pro's CALM approach uses an LLM to search external knowledge base documents"
+                }
+            ]
+        }
+        ```
+
+
+#### Example 3:
+    - **Input Answer**:
+        ```
+        Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI, in version 3.6.0.
+        ```
+    - **Ground Truth**:
+        ```
+        Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI, in version 3.7.0.
+        ```
+    - **Output**:
+        ```json
+        {
+            "statements": [
+                {
+                    "statement": "CALM is Rasa Pro's LLM-native approach to building reliable conversational AI.",
+                    "score": 1,
+                    "justification": "The ground truth confirms that Rasa Pro has released CALM, its LLM-native approach to building reliable conversational AI"
+                },
+                {
+                    "statement": "CALM was released in Rasa Pro version 3.6.0.",
+                    "score": 0,
+                    "justification": "The provided statement is incorrect, as the ground truth confirms that CALM was released in Rasa Pro version 3.7.0"
+                }
+            ]
+        }
+        ```
+
+
+## Task Inputs
+
+- **Input Answer**: {{ bot_message }}
+- **Ground Truth**: {{ ground_truth }}
+
+Your output: