rasa-pro 3.15.0a1__py3-none-any.whl → 3.15.0.dev20251027__py3-none-any.whl

rasa/builder/copilot/models.py

@@ -344,6 +344,55 @@ ChatMessage = Union[
 ]
 
 
+def create_chat_message_from_dict(message_data: Dict[str, Any]) -> ChatMessage:
+    """Parse a single chat message dictionary into a ChatMessage object.
+
+    This utility function manually parses a chat message dictionary into the
+    appropriate ChatMessage type based on its role field.
+
+    Args:
+        message_data: Dictionary containing chat message data
+
+    Returns:
+        Parsed ChatMessage object
+
+    Raises:
+        ValueError: If an unknown role is encountered
+
+    Example:
+        >>> message_data = {
+        ...     "role": "user",
+        ...     "content": [{"type": "text", "text": "Hello"}]
+        ... }
+        >>> message = parse_chat_message_from_dict(message_data)
+        >>> isinstance(message, UserChatMessage)
+        True
+        >>> message.role
+        'user'
+    """
+    available_roles = [ROLE_USER, ROLE_COPILOT, ROLE_COPILOT_INTERNAL]
+    role = message_data.get("role")
+
+    if role == ROLE_USER:
+        return UserChatMessage(**message_data)
+    elif role == ROLE_COPILOT:
+        return CopilotChatMessage(**message_data)
+    elif role == ROLE_COPILOT_INTERNAL:
+        return InternalCopilotRequestChatMessage(**message_data)
+    else:
+        message = (
+            f"Unknown role '{role}' in chat message. "
+            f"Available roles are: {', '.join(available_roles)}."
+        )
+        structlogger.error(
+            "models.create_chat_message_from_dict.unknown_role",
+            event_info=message,
+            role=role,
+            available_roles=available_roles,
+        )
+        raise ValueError(message)
+
+
 class CopilotContext(BaseModel):
     """Model containing the context used by the copilot to generate a response."""
 
@@ -391,37 +440,40 @@ class CopilotRequest(BaseModel):
 
     @field_validator("copilot_chat_history", mode="before")
     @classmethod
-    def parse_chat_history(cls, v: List[Dict[str, Any]]) -> List[ChatMessage]:
+    def parse_chat_history(
+        cls, v: Union[List[Dict[str, Any]], List[ChatMessage]]
+    ) -> List[ChatMessage]:
         """Manually parse chat history messages based on role field."""
+        # If already parsed ChatMessage objects, return them as-is
+        if (
+            v
+            and isinstance(v, list)
+            and all(isinstance(item, ChatMessage) for item in v)
+        ):
+            return v  # type: ignore[return-value]
+
+        # Check for mixed types (some ChatMessage, some not)
+        if (
+            v
+            and isinstance(v, list)
+            and any(isinstance(item, ChatMessage) for item in v)
+        ):
+            message = (
+                "Mixed types in copilot_chat_history: cannot mix ChatMessage objects"
+                "with other types."
+            )
+            structlog.get_logger().error(
+                "copilot_request.parse_chat_history.mixed_types",
+                event_info=message,
+                chat_history_types=[type(item) for item in v],
+            )
+            raise ValueError(message)
+
+        # Otherwise, parse from dictionaries
         parsed_messages: List[ChatMessage] = []
-        available_roles = [ROLE_USER, ROLE_COPILOT, ROLE_COPILOT_INTERNAL]
         for message_data in v:
-            role = message_data.get("role")
-
-            if role == ROLE_USER:
-                parsed_messages.append(UserChatMessage(**message_data))
-
-            elif role == ROLE_COPILOT:
-                parsed_messages.append(CopilotChatMessage(**message_data))
-
-            elif role == ROLE_COPILOT_INTERNAL:
-                parsed_messages.append(
-                    InternalCopilotRequestChatMessage(**message_data)
-                )
-
-            else:
-                message = (
-                    f"Unknown role '{role}' in chat message. "
-                    f"Available roles are: {', '.join(available_roles)}."
-                )
-                structlogger.error(
-                    "copilot_request.parse_chat_history.unknown_role",
-                    event_info=message,
-                    role=role,
-                    available_roles=available_roles,
-                )
-                raise ValueError(message)
-
+            chat_message = create_chat_message_from_dict(message_data)
+            parsed_messages.append(chat_message)
         return parsed_messages
 
     @property

rasa/builder/evaluator/constants.py

@@ -0,0 +1,15 @@
+"""Constants for the evaluator module."""
+
+from pathlib import Path
+
+# Base directory for the rasa package
+BASE_DIR = Path(__file__).parent.parent.parent
+
+# Response classification evaluation results directory
+RESPONSE_CLASSIFICATION_EVALUATION_RESULTS_DIR = (
+    BASE_DIR / "builder" / "evaluator" / "results"
+)
+# Default output filename
+DEFAULT_RESPONSE_CLASSIFICATION_EVALUATION_TEXT_OUTPUT_FILENAME = "run_results.txt"
+# Default YAML output filename
+RESPONSE_CLASSIFICATION_EVALUATION_YAML_OUTPUT_FILENAME = "run_results.yaml"

rasa/builder/evaluator/copilot_executor.py

@@ -0,0 +1,89 @@
+"""Copilot execution utilities for evaluators.
+
+This module provides utilities for running copilot operations in evaluation contexts,
+independent of specific evaluation frameworks like Langfuse.
+"""
+
+from typing import List, Optional
+
+import structlog
+from pydantic import BaseModel
+
+from rasa.builder.config import COPILOT_HANDLER_ROLLING_BUFFER_SIZE
+from rasa.builder.copilot.models import (
+    CopilotContext,
+    CopilotGenerationContext,
+    GeneratedContent,
+    ReferenceSection,
+    ResponseCategory,
+)
+from rasa.builder.llm_service import llm_service
+
+structlogger = structlog.get_logger()
+
+
+class CopilotRunResult(BaseModel):
+    """Result from running the copilot with response handler."""
+
+    complete_response: Optional[str]
+    response_category: Optional[ResponseCategory]
+    reference_section: Optional[ReferenceSection]
+    generation_context: CopilotGenerationContext
+
+
+async def run_copilot_with_response_handler(
+    context: CopilotContext,
+) -> Optional[CopilotRunResult]:
+    """Run the copilot with response handler on the given context.
+
+    This function encapsulates the core copilot execution logic. It handles:
+    - Instantiating the copilot and response handler
+    - Generating a response and extracting the reference section from the given context
+    - Returning structured results
+
+    Args:
+        context: The copilot context to process.
+
+    Returns:
+        CopilotRunResult containing the complete response, category, and generation
+        context, or None if execution fails.
+
+    Raises:
+        Any exceptions from the copilot or response handler execution.
+    """
+    # Instantiate the copilot and response handler
+    copilot = llm_service.instantiate_copilot()
+    copilot_response_handler = llm_service.instantiate_handler(
+        COPILOT_HANDLER_ROLLING_BUFFER_SIZE
+    )
+
+    # Call the copilot to generate a response and handle it with the response
+    # handler
+    (original_stream, generation_context) = await copilot.generate_response(context)
+    intercepted_stream = copilot_response_handler.handle_response(original_stream)
+
+    # Exhaust the stream to get the complete response for evaluation
+    response_chunks: List[str] = []
+    response_category = None
+    async for chunk in intercepted_stream:
+        if not isinstance(chunk, GeneratedContent):
+            continue
+        response_chunks.append(chunk.content)
+        response_category = chunk.response_category
+
+    complete_response = "".join(response_chunks) if response_chunks else None
+
+    # Extract the reference section from the response handler
+    if generation_context.relevant_documents:
+        reference_section = copilot_response_handler.extract_references(
+            generation_context.relevant_documents
+        )
+    else:
+        reference_section = None
+
+    return CopilotRunResult(
+        complete_response=complete_response,
+        response_category=response_category,
+        reference_section=reference_section,
+        generation_context=generation_context,
+    )

rasa/builder/evaluator/dataset/models.py

@@ -0,0 +1,173 @@
+from typing import Any, Dict, List, Optional, Union
+
+import structlog
+from pydantic import BaseModel, Field, field_validator
+
+from rasa.builder.copilot.models import (
+    ChatMessage,
+    CopilotContext,
+    EventContent,
+    ReferenceEntry,
+    ResponseCategory,
+    create_chat_message_from_dict,
+)
+from rasa.builder.document_retrieval.models import Document
+from rasa.builder.shared.tracker_context import TrackerContext
+
+structlogger = structlog.get_logger()
+
+
+class DatasetInput(BaseModel):
+    """Model for the input field of a dataset entry."""
+
+    message: Optional[str] = None
+    tracker_event_attachments: List[EventContent] = Field(default_factory=list)
+
+
+class DatasetExpectedOutput(BaseModel):
+    """Model for the expected_output field of a dataset entry."""
+
+    answer: str
+    response_category: ResponseCategory
+    references: list[ReferenceEntry]
+
+
+class DatasetMetadataCopilotAdditionalContext(BaseModel):
+    """Model for the copilot_additional_context in metadata."""
+
+    relevant_documents: List[Document] = Field(default_factory=list)
+    relevant_assistant_files: Dict[str, str] = Field(default_factory=dict)
+    assistant_tracker_context: Optional[Dict[str, Any]] = None
+    assistant_logs: str = Field(default="")
+    copilot_chat_history: List[ChatMessage] = Field(default_factory=list)
+
+    @field_validator("copilot_chat_history", mode="before")
+    @classmethod
+    def parse_chat_history(
+        cls, v: Union[List[Dict[str, Any]], List[ChatMessage]]
+    ) -> List[ChatMessage]:
+        """Manually parse chat history messages based on role field."""
+        # If already parsed ChatMessage objects, return them as-is
+        if (
+            v
+            and isinstance(v, list)
+            and all(isinstance(item, ChatMessage) for item in v)
+        ):
+            return v  # type: ignore[return-value]
+
+        # Check for mixed types (some ChatMessage, some not)
+        if (
+            v
+            and isinstance(v, list)
+            and any(isinstance(item, ChatMessage) for item in v)
+        ):
+            message = (
+                "Mixed types in copilot_chat_history: cannot mix ChatMessage objects "
+                "with other types."
+            )
+            structlogger.error(
+                "dataset_entry.parse_chat_history.mixed_types",
+                event_info=message,
+                chat_history_types=[type(item) for item in v],
+            )
+            raise ValueError(message)
+
+        # Otherwise, parse from dictionaries
+        parsed_messages: List[ChatMessage] = []
+        for message_data in v:
+            chat_message = create_chat_message_from_dict(message_data)
+            parsed_messages.append(chat_message)
+        return parsed_messages
+
+
+class DatasetMetadata(BaseModel):
+    """Model for the metadata field of a dataset entry."""
+
+    ids: Dict[str, str] = Field(default_factory=dict)
+    copilot_additional_context: DatasetMetadataCopilotAdditionalContext = Field(
+        default_factory=DatasetMetadataCopilotAdditionalContext
+    )
+
+
+class DatasetEntry(BaseModel):
+    """Pydantic model for dataset entries from Langfuse ExperimentItem."""
+
+    # Basic fields from ExperimentItem
+    id: str
+    input: DatasetInput
+    expected_output: DatasetExpectedOutput
+    metadata: DatasetMetadata
+
+    def to_copilot_context(self) -> CopilotContext:
+        """Create a CopilotContext from the dataset entry.
+
+        Raises:
+            ValueError: If the metadata is None, as it's required for creating a valid
+                CopilotContext.
+
+        Returns:
+            CopilotContext with all the context information.
+        """
+        if self.metadata is None:
+            message = (
+                f"Cannot create CopilotContext from dataset item with id: {self.id}. "
+                f"Metadata is required but was None."
+            )
+            structlogger.error(
+                "dataset_entry.to_copilot_context.metadata_is_none",
+                event_info=message,
+                item_id=self.id,
+                item_metadata=self.metadata,
+            )
+            raise ValueError(message)
+
+        # Parse tracker context if available
+        tracker_context = None
+        if (
+            self.metadata.copilot_additional_context.assistant_tracker_context
+            is not None
+        ):
+            tracker_context = TrackerContext(
+                **self.metadata.copilot_additional_context.assistant_tracker_context
+            )
+
+        return CopilotContext(
+            tracker_context=tracker_context,
+            assistant_logs=self.metadata.copilot_additional_context.assistant_logs,
+            assistant_files=self.metadata.copilot_additional_context.relevant_assistant_files,
+            copilot_chat_history=self.metadata.copilot_additional_context.copilot_chat_history,
+        )
+
+    @classmethod
+    def from_raw_data(
+        cls,
+        id: str,
+        input_data: Dict[str, Any],
+        expected_output_data: Dict[str, Any],
+        metadata_data: Dict[str, Any],
+    ) -> "DatasetEntry":
+        """Create a DatasetEntry from raw dictionary data.
+
+        Args:
+            id: The dataset entry ID.
+            input_data: Raw input dictionary.
+            expected_output_data: Raw expected output dictionary.
+            metadata_data: Raw metadata dictionary with all the additional context
+                used to generate the Copilot response.
+
+        Returns:
+            DatasetEntry with parsed data.
+        """
+        # Use Pydantic's model_validate to parse nested structures
+        dataset_input = DatasetInput.model_validate(input_data)
+        dataset_expected_output = DatasetExpectedOutput.model_validate(
+            expected_output_data
+        )
+        dataset_metadata = DatasetMetadata.model_validate(metadata_data)
+
+        return cls(
+            id=id,
+            input=dataset_input,
+            expected_output=dataset_expected_output,
+            metadata=dataset_metadata,
+        )

rasa/builder/evaluator/exceptions.py

@@ -0,0 +1,4 @@
+class EvaluationError(Exception):
+    """Base exception for evaluation-related errors."""
+
+    pass

rasa/builder/evaluator/response_classification/constants.py

@@ -0,0 +1,66 @@
+"""Constants for the response classification evaluator."""
+
+from typing import List, Literal
+
+# Options for averaging methods for Response Classification Evaluator
+MICRO_AVERAGING_METHOD: Literal["micro"] = "micro"
+MACRO_AVERAGING_METHOD: Literal["macro"] = "macro"
+WEIGHTED_AVERAGING_METHOD: Literal["weighted"] = "weighted"
+
+AVERAGING_METHODS: List[Literal["micro", "macro", "weighted"]] = [
+    MICRO_AVERAGING_METHOD,
+    MACRO_AVERAGING_METHOD,
+    WEIGHTED_AVERAGING_METHOD,
+]
+
+# Overall evaluation metric names
+MICRO_PRECISION_METRIC = "micro_precision"
+MACRO_PRECISION_METRIC = "macro_precision"
+WEIGHTED_PRECISION_METRIC = "weighted_precision"
+
+MICRO_RECALL_METRIC = "micro_recall"
+MACRO_RECALL_METRIC = "macro_recall"
+WEIGHTED_RECALL_METRIC = "weighted_recall"
+
+MICRO_F1_METRIC = "micro_f1"
+MACRO_F1_METRIC = "macro_f1"
+WEIGHTED_F1_METRIC = "weighted_f1"
+
+# Skip count metric name due to invalid data
+SKIP_COUNT_METRIC = "skipped_items"
+
+# Per-class evaluation metric name templates
+PER_CLASS_PRECISION_METRIC_TEMPLATE = "{category}_precision"
+PER_CLASS_RECALL_METRIC_TEMPLATE = "{category}_recall"
+PER_CLASS_F1_METRIC_TEMPLATE = "{category}_f1"
+PER_CLASS_SUPPORT_METRIC_TEMPLATE = "{category}_support"
+
+# Description templates for evaluation metrics
+MICRO_PRECISION_DESCRIPTION = "Micro Precision: {value:.3f}"
+MACRO_PRECISION_DESCRIPTION = "Macro Precision: {value:.3f}"
+WEIGHTED_PRECISION_DESCRIPTION = "Weighted Precision: {value:.3f}"
+
+MICRO_RECALL_DESCRIPTION = "Micro Recall: {value:.3f}"
+MACRO_RECALL_DESCRIPTION = "Macro Recall: {value:.3f}"
+WEIGHTED_RECALL_DESCRIPTION = "Weighted Recall: {value:.3f}"
+
+MICRO_F1_DESCRIPTION = "Micro F1: {value:.3f}"
+MACRO_F1_DESCRIPTION = "Macro F1: {value:.3f}"
+WEIGHTED_F1_DESCRIPTION = "Weighted F1: {value:.3f}"
+
+# Skip count metric description
+SKIP_COUNT_DESCRIPTION = "Skipped {value} items due to invalid data"
+
+# Per-class description templates
+PER_CLASS_PRECISION_DESCRIPTION = "[{category}] Precision: {value:.3f}"
+PER_CLASS_RECALL_DESCRIPTION = "[{category}] Recall: {value:.3f}"
+PER_CLASS_F1_DESCRIPTION = "[{category}] F1: {value:.3f}"
+PER_CLASS_SUPPORT_DESCRIPTION = "[{category}] Support: {value}"
+
+# Experiment configuration
+EXPERIMENT_NAME = "Copilot Response Classification Evaluation"
+EXPERIMENT_DESCRIPTION = (
+    "Evaluating Copilot response classification performance with per-class metrics "
+    "and overall averages (micro, macro, weighted). The metric that are reported are: "
+    "precision, recall, F1, support."
+)