rasa-pro 3.11.4__py3-none-any.whl → 3.12.0__py3-none-any.whl

rasa/telemetry.py

@@ -15,10 +15,11 @@ from collections import defaultdict
 from datetime import datetime
 from functools import wraps
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Text
+from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Type, cast
 
 import importlib_resources
 import requests
+import structlog
 from terminaltables import SingleTable
 
 import rasa
@@ -32,34 +33,42 @@ from rasa.constants import (
     CONFIG_TELEMETRY_ENABLED,
     CONFIG_TELEMETRY_ID,
 )
+from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.shared.constants import (
-    PROMPT_CONFIG_KEY,
-    PROMPT_TEMPLATE_CONFIG_KEY,
-    LLM_API_HEALTH_CHECK_ENV_VAR,
+    ASSISTANT_ID_KEY,
+    CONFIG_LANGUAGE_KEY,
+    CONFIG_PIPELINE_KEY,
+    CONFIG_POLICIES_KEY,
+    CONFIG_PREDICT_SCHEMA,
+    CONFIG_RECIPE_KEY,
+    CONFIG_TRAIN_SCHEMA,
+    DOCS_URL_TELEMETRY,
     LLM_API_HEALTH_CHECK_DEFAULT_VALUE,
+    LLM_API_HEALTH_CHECK_ENV_VAR,
     MODEL_GROUP_CONFIG_KEY,
+    PROMPT_CONFIG_KEY,
+    PROMPT_TEMPLATE_CONFIG_KEY,
+    UTTER_ASK_PREFIX,
 )
-from rasa.engine.storage.local_model_storage import LocalModelStorage
-from rasa.shared.constants import DOCS_URL_TELEMETRY, UTTER_ASK_PREFIX
 from rasa.shared.core.flows import Flow
 from rasa.shared.core.flows.steps import (
+    CallFlowStep,
     CollectInformationFlowStep,
-    SetSlotsFlowStep,
     LinkFlowStep,
-    CallFlowStep,
+    SetSlotsFlowStep,
 )
 from rasa.shared.exceptions import RasaException
 from rasa.utils import common as rasa_utils
 
 if typing.TYPE_CHECKING:
+    from rasa.core.agent import Agent
     from rasa.core.brokers.broker import EventBroker
-    from rasa.core.tracker_store import TrackerStore
     from rasa.core.channels.channel import InputChannel
-    from rasa.core.agent import Agent
-    from rasa.shared.nlu.training_data.training_data import TrainingData
-    from rasa.shared.importers.importer import TrainingDataImporter
+    from rasa.core.tracker_store import TrackerStore
     from rasa.core.utils import AvailableEndpoints
-    from rasa.e2e_test.e2e_test_case import TestCase, Fixture, Metadata
+    from rasa.e2e_test.e2e_test_case import Fixture, Metadata, TestCase
+    from rasa.shared.importers.importer import TrainingDataImporter
+    from rasa.shared.nlu.training_data.training_data import TrainingData
 
 logger = logging.getLogger(__name__)
 
@@ -103,6 +112,7 @@ CI_ENVIRONMENT_TELL = [
 # https://rasa.com/docs/rasa-pro/telemetry/reference
 TRAINING_STARTED_EVENT = "Training Started"
 TRAINING_COMPLETED_EVENT = "Training Completed"
+TRAINING_FAILED_EVENT = "Training Failed"
 TELEMETRY_DISABLED_EVENT = "Telemetry Disabled"
 TELEMETRY_DATA_SPLIT_EVENT = "Training Data Split"
 TELEMETRY_DATA_VALIDATED_EVENT = "Training Data Validated"
@@ -137,6 +147,8 @@ TELEMETRY_ENTERPRISE_SEARCH_POLICY_TRAINING_COMPLETED_EVENT = (
     "Enterprise Search Policy Training Completed"
 )
 TELEMETRY_ENTERPRISE_SEARCH_POLICY_PREDICT_EVENT = "Enterprise Search Policy Predicted"
+TELEMETRY_VALIDATION_ERROR_LOG_EVENT = "Validation Error Logged"
+TELEMETRY_UPLOAD_TO_STUDIO_FAILED_EVENT = "Upload to Studio Failed"
 
 # licensing events
 TELEMETRY_CONVERSATION_COUNT = "Conversation Count"
@@ -352,6 +364,7 @@ def _fetch_write_key(tool: Text, environment_variable: Text) -> Optional[Text]:
         write key, if a key was present.
     """
     import importlib_resources
+
     from rasa import __name__ as name
 
     if os.environ.get(environment_variable):
@@ -516,7 +529,6 @@ def _send_request(url: Text, payload: Dict[Text, Any]) -> None:
     """
     if _is_telemetry_debug_enabled():
         print_telemetry_payload(payload)
-        return
 
     write_key = _get_telemetry_write_key()
     if not write_key:
@@ -602,7 +614,7 @@ def _default_context_fields() -> Dict[Text, Any]:
     Return:
         A new context containing information about the runtime environment.
     """
-    from rasa.utils.licensing import property_of_active_license, get_license_hash
+    from rasa.utils.licensing import get_license_hash, property_of_active_license
 
     global TELEMETRY_CONTEXT
 
@@ -962,13 +974,14 @@ def track_model_training(
     training_id = uuid.uuid4().hex
 
     tracking_data = {
-        "language": config.get("language"),
+        "language": config.get(CONFIG_LANGUAGE_KEY),
         "training_id": training_id,
         "type": model_type,
-        "pipeline": config.get("pipeline"),
-        "policies": config.get("policies"),
-        "train_schema": config.get("train_schema"),
-        "predict_schema": config.get("predict_schema"),
+        "assistant_id": config.get(ASSISTANT_ID_KEY),
+        "pipeline": config.get(CONFIG_PIPELINE_KEY),
+        "policies": config.get(CONFIG_POLICIES_KEY),
+        "train_schema": config.get(CONFIG_TRAIN_SCHEMA),
+        "predict_schema": config.get(CONFIG_PREDICT_SCHEMA),
         "model_groups": rasa.core.utils.AvailableEndpoints.get_instance().model_groups,
         "api_health_check_enabled": (
             os.getenv(
@@ -995,7 +1008,7 @@ def track_model_training(
         "num_synonyms": len(nlu_data.entity_synonyms),
         "num_regexes": len(nlu_data.regex_features),
         "is_finetuning": is_finetuning,
-        "recipe": config.get("recipe"),
+        "recipe": config.get(CONFIG_RECIPE_KEY),
     }
 
     flow_statistics = _collect_flow_statistics(flows.underlying_flows)
@@ -1010,13 +1023,28 @@ def track_model_training(
         tracking_data,
     )
     start = datetime.now()
-    yield
+    try:
+        yield
+    except (Exception, SystemExit):
+        runtime = datetime.now() - start
+        _track(
+            TRAINING_FAILED_EVENT,
+            {
+                "training_id": training_id,
+                "assistant_id": config.get(ASSISTANT_ID_KEY),
+                "type": model_type,
+                "runtime": int(runtime.total_seconds()),
+            },
+        )
+        raise
+
     runtime = datetime.now() - start
 
     _track(
         TRAINING_COMPLETED_EVENT,
         {
             "training_id": training_id,
+            "assistant_id": config.get(ASSISTANT_ID_KEY),
             "type": model_type,
             "runtime": int(runtime.total_seconds()),
         },
@@ -1096,28 +1124,29 @@ def _get_llm_command_generator_config(config: Dict[str, Any]) -> Optional[Dict]:
     Includes the model name, whether a custom prompt is used, whether flow
     retrieval is enabled, and flow retrieval embedding model.
     """
-    from rasa.shared.constants import (
-        EMBEDDINGS_CONFIG_KEY,
-        MODEL_CONFIG_KEY,
-        MODEL_NAME_CONFIG_KEY,
-    )
     from rasa.dialogue_understanding.generator import (
+        CompactLLMCommandGenerator,
+        LLMBasedCommandGenerator,
         LLMCommandGenerator,
-        SingleStepLLMCommandGenerator,
         MultiStepLLMCommandGenerator,
-    )
-    from rasa.dialogue_understanding.generator.multi_step.multi_step_llm_command_generator import (  # noqa: E501
-        HANDLE_FLOWS_KEY,
-        FILL_SLOTS_KEY,
+        SingleStepLLMCommandGenerator,
     )
     from rasa.dialogue_understanding.generator.constants import (
-        LLM_CONFIG_KEY,
-        DEFAULT_LLM_CONFIG,
         FLOW_RETRIEVAL_KEY,
+        LLM_CONFIG_KEY,
     )
     from rasa.dialogue_understanding.generator.flow_retrieval import (
         DEFAULT_EMBEDDINGS_CONFIG,
     )
+    from rasa.dialogue_understanding.generator.multi_step.multi_step_llm_command_generator import (  # noqa: E501
+        FILL_SLOTS_KEY,
+        HANDLE_FLOWS_KEY,
+    )
+    from rasa.shared.constants import (
+        EMBEDDINGS_CONFIG_KEY,
+        MODEL_CONFIG_KEY,
+        MODEL_NAME_CONFIG_KEY,
+    )
 
     def find_command_generator_component(pipeline: List) -> Optional[Dict]:
         """Finds the LLMCommandGenerator component in the pipeline."""
@@ -1126,12 +1155,19 @@ def _get_llm_command_generator_config(config: Dict[str, Any]) -> Optional[Dict]:
                 LLMCommandGenerator.__name__,
                 SingleStepLLMCommandGenerator.__name__,
                 MultiStepLLMCommandGenerator.__name__,
+                CompactLLMCommandGenerator.__name__,
             ]:
                 return component
         return None
 
     def extract_llm_command_generator_llm_client_settings(component: Dict) -> Dict:
         """Extracts settings related to LLM command generator."""
+        component_class_lookup = {
+            LLMCommandGenerator.__name__: LLMCommandGenerator,
+            SingleStepLLMCommandGenerator.__name__: SingleStepLLMCommandGenerator,
+            MultiStepLLMCommandGenerator.__name__: MultiStepLLMCommandGenerator,
+            CompactLLMCommandGenerator.__name__: CompactLLMCommandGenerator,
+        }
         llm_config = component.get(LLM_CONFIG_KEY, {})
         # Config at this stage is not yet resolved, so read from `model_group`
         llm_model_group_id = llm_config.get(MODEL_GROUP_CONFIG_KEY)
@@ -1139,7 +1175,11 @@ def _get_llm_command_generator_config(config: Dict[str, Any]) -> Optional[Dict]:
             MODEL_NAME_CONFIG_KEY
         )
         if llm_model_group_id is None and llm_model_name is None:
-            llm_model_name = DEFAULT_LLM_CONFIG[MODEL_CONFIG_KEY]
+            component_clz = cast(
+                Type[LLMBasedCommandGenerator],
+                component_class_lookup[component["name"]],
+            )
+            llm_model_name = component_clz.get_default_llm_config()[MODEL_CONFIG_KEY]
 
         custom_prompt_used = (
             PROMPT_CONFIG_KEY in component or PROMPT_TEMPLATE_CONFIG_KEY in component
@@ -1204,7 +1244,7 @@ def _get_llm_command_generator_config(config: Dict[str, Any]) -> Optional[Dict]:
         FLOW_RETRIEVAL_EMBEDDING_MODEL_GROUP_ID: None,
     }
 
-    pipeline = config.get("pipeline", [])
+    pipeline = config.get(CONFIG_PIPELINE_KEY, [])
     if not isinstance(pipeline, list):
         return command_generator_config
 
@@ -1315,24 +1355,28 @@ def track_server_start(
     """
     from rasa.core.utils import AvailableEndpoints
 
-    def project_fingerprint_from_model(
+    def project_fingerprint_and_assistant_id_from_model(
         _model_directory: Optional[Text],
-    ) -> Optional[Text]:
+    ) -> Tuple[Optional[Text], Optional[Text]]:
         """Gets project fingerprint from an app's loaded model."""
         if not model_directory:
-            return None
+            return None, None
 
         try:
             model_archive = model.get_local_model(_model_directory)
             metadata = LocalModelStorage.metadata_from_archive(model_archive)
 
-            return metadata.project_fingerprint
+            return metadata.project_fingerprint, metadata.assistant_id
         except Exception:
-            return None
+            return None, None
 
     if not endpoints:
         endpoints = AvailableEndpoints()
 
+    project, assistant_id = project_fingerprint_and_assistant_id_from_model(
+        model_directory
+    )
+
     _track(
         TELEMETRY_SERVER_STARTED_EVENT,
         {
@@ -1354,7 +1398,8 @@ def track_server_start(
             "endpoints_event_broker": endpoints.event_broker.type
             if endpoints.event_broker
             else None,
-            "project": project_fingerprint_from_model(model_directory),
+            "project": project,
+            "assistant_id": assistant_id,
         },
     )
 
@@ -1372,24 +1417,30 @@ def track_project_init(path: Text) -> None:
 
 
 @ensure_telemetry_enabled
-def track_shell_started(model_type: Text) -> None:
+def track_shell_started(model_type: Text, assistant_id: Text) -> None:
     """Track when a user starts a bot using rasa shell.
 
     Args:
         model_type: Type of the model, core / nlu or rasa.
     """
-    _track(TELEMETRY_SHELL_STARTED_EVENT, {"type": model_type})
+    _track(
+        TELEMETRY_SHELL_STARTED_EVENT,
+        {"type": model_type, "assistant_id": assistant_id},
+    )
 
 
 @ensure_telemetry_enabled
-def track_inspect_started(model_type: Text) -> None:
+def track_inspect_started(channel: Text, assistant_id: Text) -> None:
     """Track when a user starts a bot using rasa inspect.
 
     Args:
-        channel: Channel name `socketio` (used for chat assistants)
-         or `browser_audio` (used for voice).
+        channel: Type of channel used.
+        assistant_id: ID of the assistant being inspected.
     """
-    _track(TELEMETRY_INSPECT_STARTED_EVENT, {"type": model_type})
+    _track(
+        TELEMETRY_INSPECT_STARTED_EVENT,
+        {"type": channel, "assistant_id": assistant_id},
+    )
 
 
 @ensure_telemetry_enabled
@@ -1867,3 +1918,31 @@ def track_e2e_test_conversion_completed(file_type: str, test_case_count: int) ->
             E2E_TEST_CONVERSION_TEST_CASE_COUNT: test_case_count,
         },
     )
+
+
+def track_validation_error_log(
+    _: structlog.BoundLogger, method_name: str, event_dict: Dict
+) -> Dict:
+    """Track validation errors."""
+    if is_telemetry_enabled() and method_name == "error":
+        event_dict_copy = event_dict.copy()
+
+        _track(
+            TELEMETRY_VALIDATION_ERROR_LOG_EVENT,
+            {
+                "log_level": method_name,
+                "log_id": event_dict_copy.pop("event"),
+                "message": event_dict_copy.pop("event_info", ""),
+                **event_dict_copy,
+            },
+        )
+    return event_dict
+
+
+@ensure_telemetry_enabled
+def track_upload_to_studio_failed(response_json: Dict[str, Any]) -> None:
+    """Track when the upload to studio fails."""
+    _track(
+        TELEMETRY_UPLOAD_TO_STUDIO_FAILED_EVENT,
+        {"studio_response_json": response_json},
+    )

rasa/tracing/config.py

@@ -29,9 +29,10 @@ from rasa.dialogue_understanding.commands import (
     FreeFormAnswerCommand,
 )
 from rasa.dialogue_understanding.generator import (
+    CompactLLMCommandGenerator,
     LLMCommandGenerator,
-    SingleStepLLMCommandGenerator,
     MultiStepLLMCommandGenerator,
+    SingleStepLLMCommandGenerator,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
 from rasa.dialogue_understanding.generator.nlu_command_adapter import NLUCommandAdapter
@@ -40,11 +41,11 @@ from rasa.engine.training.graph_trainer import GraphTrainer
 from rasa.tracing.constants import (
     ENDPOINTS_ENDPOINT_KEY,
     ENDPOINTS_INSECURE_KEY,
+    ENDPOINTS_METRICS_KEY,
     ENDPOINTS_OTLP_BACKEND_TYPE,
+    ENDPOINTS_ROOT_CERTIFICATES_KEY,
     ENDPOINTS_TRACING_KEY,
     ENDPOINTS_TRACING_SERVICE_NAME_KEY,
-    ENDPOINTS_ROOT_CERTIFICATES_KEY,
-    ENDPOINTS_METRICS_KEY,
 )
 from rasa.tracing.instrumentation import instrumentation
 from rasa.tracing.metric_instrument_provider import MetricInstrumentProvider
@@ -110,6 +111,7 @@ def configure_tracing(tracer_provider: Optional[TracerProvider]) -> None:
         endpoint_config_class=EndpointConfig,
         grpc_custom_action_executor_class=GRPCCustomActionExecutor,
         single_step_llm_command_generator_class=SingleStepLLMCommandGenerator,
+        compact_llm_command_generator_class=CompactLLMCommandGenerator,
         multi_step_llm_command_generator_class=MultiStepLLMCommandGenerator,
         custom_action_executor_subclasses=custom_action_executor_subclasses,
         flow_retrieval_class=FlowRetrieval,

rasa/tracing/constants.py

@@ -32,6 +32,18 @@ SINGLE_STEP_LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME = (
 SINGLE_STEP_LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME = (
     "single_step_llm_command_generator_llm_response_duration"
 )
+COMPACT_LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME = (
+    "compact_llm_command_generator_cpu_usage"
+)
+COMPACT_LLM_COMMAND_GENERATOR_MEMORY_USAGE_METRIC_NAME = (
+    "compact_llm_command_generator_memory_usage"
+)
+COMPACT_LLM_COMMAND_GENERATOR_PROMPT_TOKEN_USAGE_METRIC_NAME = (
+    "compact_llm_command_generator_prompt_token_usage"
+)
+COMPACT_LLM_COMMAND_GENERATOR_LLM_RESPONSE_DURATION_METRIC_NAME = (
+    "compact_llm_command_generator_llm_response_duration"
+)
 MULTI_STEP_LLM_COMMAND_GENERATOR_CPU_USAGE_METRIC_NAME = (
     "multi_step_llm_command_generator_cpu_usage"
 )

rasa/tracing/instrumentation/attribute_extractors.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text, Tuple, Union
 
 import tiktoken
 from numpy import ndarray
-from rasa.dialogue_understanding.generator.constants import FLOW_RETRIEVAL_KEY
 from rasa_sdk.grpc_py import action_webhook_pb2
 
 from rasa.core.actions.action import DirectCustomActionExecutor
@@ -19,34 +18,51 @@ from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephra
 from rasa.core.processor import MessageProcessor
 from rasa.core.tracker_store import TrackerStore
 from rasa.dialogue_understanding.commands import Command
-from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
 from rasa.dialogue_understanding.generator import LLMBasedCommandGenerator
+from rasa.dialogue_understanding.generator.constants import FLOW_RETRIEVAL_KEY
+from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
+from rasa.dialogue_understanding_test.du_test_result import (
+    KEY_TEST_CASES_ACCURACY,
+    KEY_USER_UTTERANCES_ACCURACY,
+    OUTPUT_COMPLETION_TOKEN_METRICS,
+    OUTPUT_LATENCY_METRICS,
+    OUTPUT_NAMES_OF_FAILED_TESTS,
+    OUTPUT_NAMES_OF_PASSED_TESTS,
+    OUTPUT_NUMBER_OF_FAILED_TESTS,
+    OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES,
+    OUTPUT_NUMBER_OF_PASSED_TESTS,
+    OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES,
+    OUTPUT_PROMPT_TOKEN_METRICS,
+    OUTPUT_TEST_CASES_ACCURACY,
+    OUTPUT_USER_UTTERANCES_ACCURACY,
+    DialogueUnderstandingTestSuiteResult,
+)
 from rasa.engine.graph import ExecutionContext, GraphModelConfiguration, GraphNode
 from rasa.engine.training.graph_trainer import GraphTrainer
 from rasa.shared.constants import (
+    CONFIG_RECIPE_KEY,
+    DEPLOYMENT_CONFIG_KEY,
     EMBEDDINGS_CONFIG_KEY,
+    LLM_CONFIG_KEY,
     MODEL_CONFIG_KEY,
+    MODEL_GROUP_ID_CONFIG_KEY,
+    MODELS_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
     TIMEOUT_CONFIG_KEY,
-    DEPLOYMENT_CONFIG_KEY,
-    MODEL_GROUP_ID_CONFIG_KEY,
-    LLM_CONFIG_KEY,
 )
 from rasa.shared.core.constants import REQUESTED_SLOT
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.events import DialogueStackUpdated, Event
 from rasa.shared.core.flows import Flow, FlowsList, FlowStep
 from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.core.training_data.structures import StoryGraph
 from rasa.shared.importers.importer import TrainingDataImporter
 from rasa.shared.nlu.constants import INTENT_NAME_KEY, SET_SLOT_COMMAND
-from rasa.shared.utils.llm import (
-    combine_custom_and_default_config,
-)
+from rasa.shared.utils.llm import combine_custom_and_default_config
 from rasa.tracing.constants import (
     PROMPT_TOKEN_LENGTH_ATTRIBUTE_NAME,
     REQUEST_BODY_SIZE_IN_BYTES_ATTRIBUTE_NAME,
 )
-from rasa.shared.core.training_data.structures import StoryGraph
 
 if TYPE_CHECKING:
     from langchain.llms.base import BaseLLM
@@ -255,7 +271,7 @@ def extract_attrs_for_graph_trainer(
     return {
         "training_type": model_configuration.training_type.model_type,
         "language": model_configuration.language,
-        "recipe_name": importer.get_config().get("recipe"),
+        "recipe_name": importer.get_config().get(CONFIG_RECIPE_KEY),
         "output_filename": output_filename.name,
         "is_finetuning": is_finetuning,
     }
@@ -357,14 +373,13 @@ def extract_attrs_for_llm_based_command_generator(
     self: "LLMBasedCommandGenerator",
     prompt: str,
 ) -> Dict[str, Any]:
-    from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
     from rasa.dialogue_understanding.generator.flow_retrieval import (
         DEFAULT_EMBEDDINGS_CONFIG,
     )
 
     attributes = extract_llm_config(
         self,
-        default_llm_config=DEFAULT_LLM_CONFIG,
+        default_llm_config=self.get_default_llm_config(),
         default_embeddings_config=DEFAULT_EMBEDDINGS_CONFIG,
     )
 
@@ -570,6 +585,67 @@ def extract_attrs_for_advance_flows(
     }
 
 
+def extract_attrs_for_du_print_test_results(
+    test_suite_result: DialogueUnderstandingTestSuiteResult,
+    output_prompt: bool,
+) -> Dict[str, Any]:
+    """Extract the attributes for
+    `rasa.dialogue_understanding_test.io.print_test_results` function.
+    """
+    from rasa.tracing.instrumentation.instrumentation import (
+        DIALOG_UNDERSTANDING_TEST_IO_MODULE_NAME,
+    )
+
+    attributes_dict = {
+        "module_name": DIALOG_UNDERSTANDING_TEST_IO_MODULE_NAME,
+        OUTPUT_TEST_CASES_ACCURACY: test_suite_result.accuracy[KEY_TEST_CASES_ACCURACY],
+        OUTPUT_USER_UTTERANCES_ACCURACY: test_suite_result.accuracy[
+            KEY_USER_UTTERANCES_ACCURACY
+        ],
+        OUTPUT_NUMBER_OF_PASSED_TESTS: test_suite_result.number_of_passed_tests,
+        OUTPUT_NUMBER_OF_FAILED_TESTS: test_suite_result.number_of_failed_tests,
+        OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES: test_suite_result.number_of_passed_user_utterances,  # noqa: E501
+        OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES: test_suite_result.number_of_failed_user_utterances,  # noqa: E501
+        OUTPUT_NAMES_OF_PASSED_TESTS: json.dumps(
+            test_suite_result.names_of_passed_tests
+        ),
+        OUTPUT_NAMES_OF_FAILED_TESTS: json.dumps(
+            test_suite_result.names_of_failed_tests
+        ),
+    }
+    if test_suite_result.command_metrics:
+        for (
+            command_name,
+            command_metric,
+        ) in test_suite_result.command_metrics.items():
+            # OpenTelemetry / Honeycomb doesn't support dictionaries/json values,
+            # so we need to set the values as separate attributes
+            for metric_name, value in command_metric.as_dict().items():
+                attributes_dict[f"{command_name}_{metric_name}"] = value
+
+    if test_suite_result.llm_config:
+        # check if model group syntax is used
+        if MODELS_CONFIG_KEY in test_suite_result.llm_config:
+            for idx, model_group in enumerate(
+                test_suite_result.llm_config[MODELS_CONFIG_KEY]
+            ):
+                for key, value in model_group.items():
+                    if value is not None:
+                        attributes_dict[f"llm_config_{idx}_{key}"] = value
+        else:
+            for key, value in test_suite_result.llm_config.items():
+                attributes_dict[f"llm_config_0_{key}"] = value
+
+    for key, value in test_suite_result.latency_metrics.items():
+        attributes_dict[f"{OUTPUT_LATENCY_METRICS}_{key}"] = value
+    for key, value in test_suite_result.prompt_token_metrics.items():
+        attributes_dict[f"{OUTPUT_PROMPT_TOKEN_METRICS}_{key}"] = value
+    for key, value in test_suite_result.completion_token_metrics.items():
+        attributes_dict[f"{OUTPUT_COMPLETION_TOKEN_METRICS}_{key}"] = value
+
+    return attributes_dict
+
+
 def extract_attrs_for_run_step(
     step: FlowStep,
     flow: Flow,
@@ -577,6 +653,7 @@ def extract_attrs_for_run_step(
     tracker: DialogueStateTracker,
     available_actions: List[str],
     flows: FlowsList,
+    previous_step_id: Text,
 ) -> Dict[str, Any]:
     current_context = extract_current_context_attribute(stack)
 
@@ -585,6 +662,7 @@ def extract_attrs_for_run_step(
         "step_description": step.description if step.description else "None",
         "current_flow_id": flow.id,
         "current_context": json.dumps(current_context),
+        "previous_step_id": previous_step_id,
     }
 
 
@@ -644,8 +722,8 @@ def extract_attrs_for_intentless_policy_generate_llm_answer(
     self: "IntentlessPolicy", llm: "BaseLLM", prompt: str
 ) -> Dict[str, Any]:
     from rasa.core.policies.intentless_policy import (
-        DEFAULT_LLM_CONFIG,
         DEFAULT_EMBEDDINGS_CONFIG,
+        DEFAULT_LLM_CONFIG,
     )
 
     attributes = extract_llm_config(
@@ -661,8 +739,8 @@ def extract_attrs_for_enterprise_search_generate_llm_answer(
     self: "EnterpriseSearchPolicy", llm: "BaseLLM", prompt: str
 ) -> Dict[str, Any]:
     from rasa.core.policies.enterprise_search_policy import (
-        DEFAULT_LLM_CONFIG,
         DEFAULT_EMBEDDINGS_CONFIG,
+        DEFAULT_LLM_CONFIG,
     )
 
     attributes = extract_llm_config(