rasa-pro 3.12.21__py3-none-any.whl → 3.13.0__py3-none-any.whl

rasa/core/policies/enterprise_search_policy.py

@@ -1,3 +1,4 @@
+import dataclasses
 import glob
 import importlib.resources
 import json
@@ -11,10 +12,8 @@ from jinja2 import Template
 from pydantic import ValidationError
 
 import rasa.shared.utils.io
+from rasa.core.available_endpoints import AvailableEndpoints
 from rasa.core.constants import (
-    POLICY_MAX_HISTORY,
-    POLICY_PRIORITY,
-    SEARCH_POLICY_PRIORITY,
     UTTER_SOURCE_METADATA_KEY,
 )
 from rasa.core.information_retrieval import (
@@ -24,8 +23,15 @@ from rasa.core.information_retrieval import (
     create_from_endpoint_config,
 )
 from rasa.core.information_retrieval.faiss import FAISS_Store
+from rasa.core.policies.enterprise_search_policy_config import (
+    DEFAULT_EMBEDDINGS_CONFIG,
+    DEFAULT_ENTERPRISE_SEARCH_CONFIG,
+    DEFAULT_LLM_CONFIG,
+    DEFAULT_VECTOR_STORE_TYPE,
+    SOURCE_PROPERTY,
+    EnterpriseSearchPolicyConfig,
+)
 from rasa.core.policies.policy import Policy, PolicyPrediction
-from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
 )
@@ -48,16 +54,11 @@ from rasa.graph_components.providers.forms_provider import Forms
 from rasa.graph_components.providers.responses_provider import Responses
 from rasa.shared.constants import (
     EMBEDDINGS_CONFIG_KEY,
-    MAX_COMPLETION_TOKENS_CONFIG_KEY,
-    MAX_RETRIES_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_GROUP_ID_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
-    OPENAI_PROVIDER,
-    PROMPT_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
-    TEMPERATURE_CONFIG_KEY,
-    TIMEOUT_CONFIG_KEY,
+    RASA_PATTERN_CANNOT_HANDLE_NO_RELEVANT_ANSWER,
 )
 from rasa.shared.core.constants import (
     ACTION_CANCEL_FLOW,
@@ -80,7 +81,6 @@ from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.providers.embedding._langchain_embedding_client_adapter import (
     _LangchainEmbeddingClientAdapter,
 )
-from rasa.shared.providers.llm.llm_client import LLMClient
 from rasa.shared.providers.llm.llm_response import LLMResponse, measure_llm_latency
 from rasa.shared.utils.cli import print_error_and_exit
 from rasa.shared.utils.constants import (
@@ -93,12 +93,9 @@ from rasa.shared.utils.health_check.embeddings_health_check_mixin import (
 from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.io import deep_container_fingerprint
 from rasa.shared.utils.llm import (
-    DEFAULT_OPENAI_CHAT_MODEL_NAME,
-    DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
     embedder_factory,
     get_prompt_template,
     llm_factory,
-    resolve_model_client_config,
     sanitize_message_for_prompt,
     tracker_as_readable_transcript,
 )
@@ -115,41 +112,10 @@ if TYPE_CHECKING:
 
 from rasa.utils.log_utils import log_llm
 
-logger = structlog.get_logger()
+structlogger = structlog.get_logger()
 
 dotenv.load_dotenv("./.env")
 
-SOURCE_PROPERTY = "source"
-VECTOR_STORE_TYPE_PROPERTY = "type"
-VECTOR_STORE_PROPERTY = "vector_store"
-VECTOR_STORE_THRESHOLD_PROPERTY = "threshold"
-TRACE_TOKENS_PROPERTY = "trace_prompt_tokens"
-CITATION_ENABLED_PROPERTY = "citation_enabled"
-USE_LLM_PROPERTY = "use_generative_llm"
-MAX_MESSAGES_IN_QUERY_KEY = "max_messages_in_query"
-
-DEFAULT_VECTOR_STORE_TYPE = "faiss"
-DEFAULT_VECTOR_STORE_THRESHOLD = 0.0
-DEFAULT_VECTOR_STORE = {
-    VECTOR_STORE_TYPE_PROPERTY: DEFAULT_VECTOR_STORE_TYPE,
-    SOURCE_PROPERTY: "./docs",
-    VECTOR_STORE_THRESHOLD_PROPERTY: DEFAULT_VECTOR_STORE_THRESHOLD,
-}
-
-DEFAULT_LLM_CONFIG = {
-    PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
-    MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME,
-    TIMEOUT_CONFIG_KEY: 10,
-    TEMPERATURE_CONFIG_KEY: 0.0,
-    MAX_COMPLETION_TOKENS_CONFIG_KEY: 256,
-    MAX_RETRIES_CONFIG_KEY: 1,
-}
-
-DEFAULT_EMBEDDINGS_CONFIG = {
-    PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
-    MODEL_CONFIG_KEY: DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
-}
-
 ENTERPRISE_SEARCH_PROMPT_FILE_NAME = "enterprise_search_policy_prompt.jinja2"
 ENTERPRISE_SEARCH_CONFIG_FILE_NAME = "config.json"
 
@@ -164,6 +130,15 @@ DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_CITATION_TEMPLATE = importlib.resources.re
     "rasa.core.policies", "enterprise_search_prompt_with_citation_template.jinja2"
 )
 
+DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_RELEVANCY_CHECK_AND_CITATION_TEMPLATE = (
+    importlib.resources.read_text(
+        "rasa.core.policies",
+        "enterprise_search_prompt_with_relevancy_check_and_citation_template.jinja2",
+    )
+)
+
+_ENTERPRISE_SEARCH_ANSWER_NOT_RELEVANT_PATTERN = re.compile(r"\[NO_RAG_ANSWER\]")
+
 _ENTERPRISE_SEARCH_CITATION_PATTERN = re.compile(r"\[([^\]]+)\]")
 
 
@@ -175,6 +150,12 @@ class VectorStoreConfigurationError(RasaException):
     """Exception raised for errors in vector store configuration."""
 
 
+@dataclasses.dataclass
+class _RelevancyCheckResponse:
+    answer: Optional[str]
+    relevant: bool
+
+
 @DefaultV1Recipe.register(
     DefaultV1Recipe.ComponentType.POLICY_WITH_END_TO_END_SUPPORT, is_trainable=True
 )
@@ -206,10 +187,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
     @staticmethod
     def get_default_config() -> Dict[str, Any]:
         """Returns the default config of the policy."""
-        return {
-            POLICY_PRIORITY: SEARCH_POLICY_PRIORITY,
-            VECTOR_STORE_PROPERTY: DEFAULT_VECTOR_STORE,
-        }
+        return DEFAULT_ENTERPRISE_SEARCH_CONFIG
 
     def __init__(
         self,
@@ -224,76 +202,71 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         """Constructs a new Policy object."""
         super().__init__(config, model_storage, resource, execution_context, featurizer)
 
-        # Resolve LLM config
-        self.config[LLM_CONFIG_KEY] = resolve_model_client_config(
-            self.config.get(LLM_CONFIG_KEY), EnterpriseSearchPolicy.__name__
-        )
-        # Resolve embeddings config
-        self.config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
-            self.config.get(EMBEDDINGS_CONFIG_KEY), EnterpriseSearchPolicy.__name__
-        )
+        parsed_config = EnterpriseSearchPolicyConfig.from_dict(config)
 
         # Vector store object and configuration
         self.vector_store = vector_store
-        self.vector_store_config = self.config.get(
-            VECTOR_STORE_PROPERTY, DEFAULT_VECTOR_STORE
-        )
+        self.vector_store_config = parsed_config.vector_store_config
+        self.vector_search_threshold = parsed_config.vector_store_threshold
+        self.vector_store_type = parsed_config.vector_store_type
 
-        # Embeddings configuration for encoding the search query
-        self.embeddings_config = (
-            self.config[EMBEDDINGS_CONFIG_KEY] or DEFAULT_EMBEDDINGS_CONFIG
-        )
+        # Resolved embeddings configuration for encoding the search query
+        self.embeddings_config = parsed_config.embeddings_config
 
-        # LLM Configuration for response generation
-        self.llm_config = self.config[LLM_CONFIG_KEY] or DEFAULT_LLM_CONFIG
+        # Resolved LLM Configuration for response generation
+        self.llm_config = parsed_config.llm_config
 
         # Maximum number of turns to include in the prompt
-        self.max_history = self.config.get(POLICY_MAX_HISTORY)
+        self.max_history = parsed_config.max_history
 
         # Maximum number of messages to include in the search query
-        self.max_messages_in_query = self.config.get(MAX_MESSAGES_IN_QUERY_KEY, 2)
+        self.max_messages_in_query = parsed_config.max_messages_in_query
+
+        # Boolean to enable/disable tracing of prompt tokens
+        self.trace_prompt_tokens = parsed_config.trace_prompt_tokens
 
-        # boolean to enable/disable tracing of prompt tokens
-        self.trace_prompt_tokens = self.config.get(TRACE_TOKENS_PROPERTY, False)
+        # Boolean to enable/disable the use of LLM for response generation
+        self.use_llm = parsed_config.use_generative_llm
 
-        # boolean to enable/disable the use of LLM for response generation
-        self.use_llm = self.config.get(USE_LLM_PROPERTY, True)
+        # Boolean to enable/disable citation generation. This flag enables citation
+        # logic, but it only takes effect if `use_llm` is True.
+        self.citation_enabled = parsed_config.enable_citation
 
-        # boolean to enable/disable citation generation
-        self.citation_enabled = self.config.get(CITATION_ENABLED_PROPERTY, False)
+        # Boolean to enable/disable the use of relevancy check alongside answer
+        # generation. This flag enables citation logic, but it only takes effect if
+        # `use_llm` is True.
+        self.relevancy_check_enabled = parsed_config.check_relevancy
 
+        # Resolve the prompt template. The prompt will only be used if the 'use_llm' is
+        # set to True.
         self.prompt_template = prompt_template or get_prompt_template(
-            self.config.get(PROMPT_CONFIG_KEY),
-            DEFAULT_ENTERPRISE_SEARCH_PROMPT_TEMPLATE,
-            log_source_component=EnterpriseSearchPolicy.__name__,
-            log_source_method=LOG_COMPONENT_SOURCE_METHOD_INIT,
-        )
-        self.citation_prompt_template = get_prompt_template(
-            self.config.get(PROMPT_CONFIG_KEY),
-            DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_CITATION_TEMPLATE,
+            jinja_file_path=parsed_config.prompt_template,
+            default_prompt_template=self._select_default_prompt_template_based_on_features(
+                parsed_config.check_relevancy, parsed_config.enable_citation
+            ),
             log_source_component=EnterpriseSearchPolicy.__name__,
             log_source_method=LOG_COMPONENT_SOURCE_METHOD_INIT,
         )
-        # If citation is enabled, use the citation prompt template
-        if self.citation_enabled:
-            self.prompt_template = self.citation_prompt_template
 
     @classmethod
-    def _create_plain_embedder(cls, config: Dict[Text, Any]) -> "Embeddings":
+    def _create_plain_embedder(cls, embeddings_config: Dict[Text, Any]) -> "Embeddings":
         """Creates an embedder based on the given configuration.
 
+        Args:
+            embeddings_config: A resolved embeddings configuration. Resolved means the
+            configuration is either:
+                - A reference to a model group that has already been expanded into
+                  its corresponding configuration using the information from
+                  `endpoints.yml`, or
+                - A full configuration for the embedder defined directly (i.e. not
+                  relying on model groups or indirections).
+
         Returns:
-        The embedder.
+            The embedder.
         """
         # Copy the config so original config is not modified
-        config = config.copy()
-        # Resolve config and instantiate the embedding client
-        config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
-            config.get(EMBEDDINGS_CONFIG_KEY), EnterpriseSearchPolicy.__name__
-        )
-        client = embedder_factory(
-            config.get(EMBEDDINGS_CONFIG_KEY), DEFAULT_EMBEDDINGS_CONFIG
-        )
+        embeddings_config = embeddings_config.copy()
+        client = embedder_factory(embeddings_config, DEFAULT_EMBEDDINGS_CONFIG)
         # Wrap the embedding client in the adapter
         return _LangchainEmbeddingClientAdapter(client)
 
@@ -359,18 +332,18 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             can load the policy from the resource.
         """
         # Perform health checks for both LLM and embeddings client configs
-        self._perform_health_checks(self.config, "enterprise_search_policy.train")
-
-        store_type = self.vector_store_config.get(VECTOR_STORE_TYPE_PROPERTY)
+        self._perform_health_checks(
+            self.llm_config, self.embeddings_config, "enterprise_search_policy.train"
+        )
 
         # telemetry call to track training start
         track_enterprise_search_policy_train_started()
 
         # validate embedding configuration
         try:
-            embeddings = self._create_plain_embedder(self.config)
+            embeddings = self._create_plain_embedder(self.embeddings_config)
         except (ValidationError, Exception) as e:
-            logger.error(
+            structlogger.error(
                 "enterprise_search_policy.train.embedder_instantiation_failed",
                 message="Unable to instantiate the embedding client.",
                 error=e,
@@ -380,8 +353,8 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                 f"required environment variables. Error: {e}"
             )
 
-        if store_type == DEFAULT_VECTOR_STORE_TYPE:
-            logger.info("enterprise_search_policy.train.faiss")
+        if self.vector_store_type == DEFAULT_VECTOR_STORE_TYPE:
+            structlogger.info("enterprise_search_policy.train.faiss")
             docs_folder = self.vector_store_config.get(SOURCE_PROPERTY)
             self._validate_documents_folder(docs_folder)
             with self._model_storage.write_to(self._resource) as path:
@@ -390,13 +363,17 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                     embeddings=embeddings,
                     index_path=path,
                     create_index=True,
+                    parse_as_faq_pairs=not self.use_llm,
                 )
         else:
-            logger.info("enterprise_search_policy.train.custom", store_type=store_type)
+            structlogger.info(
+                "enterprise_search_policy.train.custom",
+                store_type=self.vector_store_type,
+            )
 
         # telemetry call to track training completion
         track_enterprise_search_policy_train_completed(
-            vector_store_type=store_type,
+            vector_store_type=self.vector_store_type,
             embeddings_type=self.embeddings_config.get(PROVIDER_CONFIG_KEY),
             embeddings_model=self.embeddings_config.get(MODEL_CONFIG_KEY)
             or self.embeddings_config.get(MODEL_NAME_CONFIG_KEY),
@@ -408,6 +385,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             or self.llm_config.get(MODEL_NAME_CONFIG_KEY),
             llm_model_group_id=self.llm_config.get(MODEL_GROUP_ID_CONFIG_KEY),
             citation_enabled=self.citation_enabled,
+            relevancy_check_enabled=self.relevancy_check_enabled,
         )
         self.persist()
         return self._resource
@@ -418,8 +396,11 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             rasa.shared.utils.io.write_text_file(
                 self.prompt_template, path / ENTERPRISE_SEARCH_PROMPT_FILE_NAME
             )
+            config = self.config.copy()
+            config[LLM_CONFIG_KEY] = self.llm_config
+            config[EMBEDDINGS_CONFIG_KEY] = self.embeddings_config
             rasa.shared.utils.io.dump_obj_as_json_to_file(
-                path / ENTERPRISE_SEARCH_CONFIG_FILE_NAME, self.config
+                path / ENTERPRISE_SEARCH_CONFIG_FILE_NAME, config
             )
 
     def _prepare_slots_for_template(
@@ -458,9 +439,8 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             endpoints: Endpoints configuration.
         """
         config = endpoints.vector_store if endpoints else None
-        store_type = self.vector_store_config.get(VECTOR_STORE_TYPE_PROPERTY)
-        if config is None and store_type != DEFAULT_VECTOR_STORE_TYPE:
-            logger.error(
+        if config is None and self.vector_store_type != DEFAULT_VECTOR_STORE_TYPE:
+            structlogger.error(
                 "enterprise_search_policy._connect_vector_store_or_raise.no_config"
             )
             raise VectorStoreConfigurationError(
@@ -470,7 +450,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         try:
             self.vector_store.connect(config)  # type: ignore
         except Exception as e:
-            logger.error(
+            structlogger.error(
                 "enterprise_search_policy._connect_vector_store_or_raise.connect_error",
                 error=e,
                 config=config,
@@ -496,14 +476,14 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                 transcript.append(sanitize_message_for_prompt(event.text))
 
         search_query = " ".join(transcript[-history:][::-1])
-        logger.debug("search_query", search_query=search_query)
+        structlogger.debug("search_query", search_query=search_query)
         return search_query
 
     async def predict_action_probabilities(  # type: ignore[override]
         self,
         tracker: DialogueStateTracker,
         domain: Domain,
-        endpoints: Optional[AvailableEndpoints],
+        endpoints: Optional[AvailableEndpoints] = None,
         rule_only_data: Optional[Dict[Text, Any]] = None,
         **kwargs: Any,
     ) -> PolicyPrediction:
@@ -522,23 +502,20 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
              The prediction.
         """
         logger_key = "enterprise_search_policy.predict_action_probabilities"
-        vector_search_threshold = self.vector_store_config.get(
-            VECTOR_STORE_THRESHOLD_PROPERTY, DEFAULT_VECTOR_STORE_THRESHOLD
-        )
-        llm = llm_factory(self.config.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG)
+
         if not self.supports_current_stack_frame(
             tracker, False, False
         ) or self.should_abstain_in_coexistence(tracker, True):
             return self._prediction(self._default_predictions(domain))
 
         if not self.vector_store:
-            logger.error(f"{logger_key}.no_vector_store")
+            structlogger.error(f"{logger_key}.no_vector_store")
             return self._create_prediction_internal_error(domain, tracker)
 
         try:
             self._connect_vector_store_or_raise(endpoints)
         except (VectorStoreConfigurationError, VectorStoreConnectionError) as e:
-            logger.error(f"{logger_key}.connection_error", error=e)
+            structlogger.error(f"{logger_key}.connection_error", error=e)
             return self._create_prediction_internal_error(domain, tracker)
 
         search_query = self._prepare_search_query(
@@ -550,20 +527,19 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             documents = await self.vector_store.search(
                 query=search_query,
                 tracker_state=tracker_state,
-                threshold=vector_search_threshold,
+                threshold=self.vector_search_threshold,
             )
         except InformationRetrievalException as e:
-            logger.error(f"{logger_key}.search_error", error=e)
+            structlogger.error(f"{logger_key}.search_error", error=e)
             return self._create_prediction_internal_error(domain, tracker)
 
         if not documents.results:
-            logger.info(f"{logger_key}.no_documents")
+            structlogger.info(f"{logger_key}.no_documents")
             return self._create_prediction_cannot_handle(domain, tracker)
 
         if self.use_llm:
             prompt = self._render_prompt(tracker, documents.results)
-            llm_response = await self._generate_llm_answer(llm, prompt)
-            llm_response = LLMResponse.ensure_llm_response(llm_response)
+            llm_response = await self._invoke_llm(prompt)
 
             self._add_prompt_and_llm_response_to_latest_message(
                 tracker=tracker,
@@ -573,24 +549,38 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             )
 
             if llm_response is None or not llm_response.choices:
-                logger.debug(f"{logger_key}.no_llm_response")
+                structlogger.debug(f"{logger_key}.no_llm_response")
                 response = None
             else:
                 llm_answer = llm_response.choices[0]
 
+                if self.relevancy_check_enabled:
+                    relevancy_response = self._parse_llm_relevancy_check_response(
+                        llm_answer
+                    )
+                    if not relevancy_response.relevant:
+                        structlogger.debug(f"{logger_key}.answer_not_relevant")
+                        return self._create_prediction_cannot_handle(
+                            domain,
+                            tracker,
+                            RASA_PATTERN_CANNOT_HANDLE_NO_RELEVANT_ANSWER,
+                        )
+
                 if self.citation_enabled:
                     llm_answer = self.post_process_citations(llm_answer)
 
-                logger.debug(f"{logger_key}.llm_answer", llm_answer=llm_answer)
+                structlogger.debug(
+                    f"{logger_key}.llm_answer", prompt=prompt, llm_answer=llm_answer
+                )
                 response = llm_answer
         else:
             response = documents.results[0].metadata.get("answer", None)
             if not response:
-                logger.error(
+                structlogger.error(
                     f"{logger_key}.answer_key_missing_in_metadata",
                     search_results=documents.results,
                 )
-            logger.debug(
+            structlogger.debug(
                 "enterprise_search_policy.predict_action_probabilities.no_llm",
                 search_results=documents,
             )
@@ -610,7 +600,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
         # telemetry call to track policy prediction
         track_enterprise_search_policy_predict(
-            vector_store_type=self.vector_store_config.get(VECTOR_STORE_TYPE_PROPERTY),
+            vector_store_type=self.vector_store_type,
             embeddings_type=self.embeddings_config.get(PROVIDER_CONFIG_KEY),
             embeddings_model=self.embeddings_config.get(MODEL_CONFIG_KEY)
             or self.embeddings_config.get(MODEL_NAME_CONFIG_KEY),
@@ -622,6 +612,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             or self.llm_config.get(MODEL_NAME_CONFIG_KEY),
             llm_model_group_id=self.llm_config.get(MODEL_GROUP_ID_CONFIG_KEY),
             citation_enabled=self.citation_enabled,
+            relevancy_check_enabled=self.relevancy_check_enabled,
         )
         return self._create_prediction(
             domain=domain, tracker=tracker, action_metadata=action_metadata
@@ -645,11 +636,12 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             ),
             "docs": documents,
             "slots": self._prepare_slots_for_template(tracker),
+            "check_relevancy": self.relevancy_check_enabled,
             "citation_enabled": self.citation_enabled,
         }
         prompt = Template(self.prompt_template).render(**inputs)
         log_llm(
-            logger=logger,
+            logger=structlogger,
             log_module="EnterpriseSearchPolicy",
             log_event="enterprise_search_policy._render_prompt.prompt_rendered",
             prompt=prompt,
@@ -657,9 +649,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         return prompt
 
     @measure_llm_latency
-    async def _generate_llm_answer(
-        self, llm: LLMClient, prompt: Text
-    ) -> Optional[LLMResponse]:
+    async def _invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
         """Fetches an LLM completion for the provided prompt.
 
         Args:
@@ -669,17 +659,32 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         Returns:
             An LLMResponse object, or None if the call fails.
         """
+        llm = llm_factory(self.llm_config, DEFAULT_LLM_CONFIG)
         try:
-            return await llm.acompletion(prompt)
+            response = await llm.acompletion(prompt)
+            return LLMResponse.ensure_llm_response(response)
         except Exception as e:
             # unfortunately, langchain does not wrap LLM exceptions which means
             # we have to catch all exceptions here
-            logger.error(
+            structlogger.error(
                 "enterprise_search_policy._generate_llm_answer.llm_error",
                 error=e,
             )
             return None
 
+    def _parse_llm_relevancy_check_response(
+        self, llm_answer: str
+    ) -> _RelevancyCheckResponse:
+        """Checks if the LLM response is relevant by parsing it."""
+        answer_relevant = not _ENTERPRISE_SEARCH_ANSWER_NOT_RELEVANT_PATTERN.search(
+            llm_answer
+        )
+        structlogger.debug("")
+        return _RelevancyCheckResponse(
+            answer=llm_answer if answer_relevant else None,
+            relevant=answer_relevant,
+        )
+
     def _create_prediction(
         self,
         domain: Domain,
@@ -714,10 +719,18 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         )
 
     def _create_prediction_cannot_handle(
-        self, domain: Domain, tracker: DialogueStateTracker
+        self,
+        domain: Domain,
+        tracker: DialogueStateTracker,
+        reason: Optional[str] = None,
     ) -> PolicyPrediction:
+        cannot_handle_stack_frame = (
+            CannotHandlePatternFlowStackFrame(reason=reason)
+            if reason is not None
+            else CannotHandlePatternFlowStackFrame()
+        )
         return self._create_prediction_for_pattern(
-            domain, tracker, CannotHandlePatternFlowStackFrame()
+            domain, tracker, cannot_handle_stack_frame
         )
 
     def _create_prediction_for_pattern(
@@ -775,7 +788,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                 "Please specify a valid path to the documents source directory in the "
                 "vector_store configuration."
             )
-            logger.error(
+            structlogger.error(
                 "enterprise_search_policy.train.faiss.invalid_source_directory",
                 message=error_message,
             )
@@ -787,7 +800,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                 f"Document source directory is empty: '{docs_folder}'. "
                 "Please add documents to this directory or specify a different one."
             )
-            logger.error(
+            structlogger.error(
                 "enterprise_search_policy.train.faiss.source_directory_empty",
                 message=error_message,
             )
@@ -803,70 +816,93 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         **kwargs: Any,
     ) -> "EnterpriseSearchPolicy":
         """Loads a trained policy (see parent class for full docstring)."""
+        parsed_config = EnterpriseSearchPolicyConfig.from_dict(config)
+
         # Perform health checks for both LLM and embeddings client configs
-        cls._perform_health_checks(config, "enterprise_search_policy.load")
+        cls._perform_health_checks(
+            parsed_config.llm_config,
+            parsed_config.embeddings_config,
+            "enterprise_search_policy.load",
+        )
 
-        prompt_template = None
+        prompt_template = cls._load_prompt_template(model_storage, resource)
+        embeddings = cls._create_plain_embedder(parsed_config.embeddings_config)
+        vector_store = cls._load_vector_store(
+            embeddings,
+            parsed_config.vector_store_type,
+            parsed_config.use_generative_llm,
+            model_storage,
+            resource,
+        )
+
+        structlogger.info("enterprise_search_policy.load", config=config)
+
+        return cls(
+            config,
+            model_storage,
+            resource,
+            execution_context,
+            vector_store=vector_store,
+            prompt_template=prompt_template,
+        )
+
+    @classmethod
+    def _load_prompt_template(
+        cls, model_storage: ModelStorage, resource: Resource
+    ) -> Optional[str]:
         try:
             with model_storage.read_from(resource) as path:
-                prompt_template = rasa.shared.utils.io.read_file(
+                return rasa.shared.utils.io.read_file(
                     path / ENTERPRISE_SEARCH_PROMPT_FILE_NAME
                 )
         except (FileNotFoundError, FileIOException) as e:
-            logger.warning(
+            structlogger.warning(
                 "enterprise_search_policy.load.failed", error=e, resource=resource.name
             )
+        return None
 
-        store_type = config.get(VECTOR_STORE_PROPERTY, {}).get(
-            VECTOR_STORE_TYPE_PROPERTY
-        )
-
-        embeddings = cls._create_plain_embedder(config)
-
-        logger.info("enterprise_search_policy.load", config=config)
+    @classmethod
+    def _load_vector_store(
+        cls,
+        embeddings: "Embeddings",
+        store_type: str,
+        use_generative_llm: bool,
+        model_storage: ModelStorage,
+        resource: Resource,
+    ) -> InformationRetrieval:
         if store_type == DEFAULT_VECTOR_STORE_TYPE:
             # if a vector store is not specified,
             # default to using FAISS with the index stored in the model
             # TODO figure out a way to get path without context manager
             with model_storage.read_from(resource) as path:
-                vector_store = FAISS_Store(
+                return FAISS_Store(
                     embeddings=embeddings,
                     index_path=path,
                     docs_folder=None,
                     create_index=False,
+                    parse_as_faq_pairs=not use_generative_llm,
                 )
         else:
-            vector_store = create_from_endpoint_config(
+            return create_from_endpoint_config(
                 config_type=store_type,
                 embeddings=embeddings,
-            )  # type: ignore
-
-        return cls(
-            config,
-            model_storage,
-            resource,
-            execution_context,
-            vector_store=vector_store,
-            prompt_template=prompt_template,
-        )
+            )
 
     @classmethod
-    def _get_local_knowledge_data(cls, config: Dict[str, Any]) -> Optional[List[str]]:
+    def _get_local_knowledge_data(
+        cls, store_type: str, source: Optional[str] = None
+    ) -> Optional[List[str]]:
         """This is required only for local knowledge base types.
 
         e.g. FAISS, to ensure that the graph component is retrained when the knowledge
         base is updated.
         """
-        merged_config = {**cls.get_default_config(), **config}
-
-        store_type = merged_config.get(VECTOR_STORE_PROPERTY, {}).get(
-            VECTOR_STORE_TYPE_PROPERTY
-        )
-        if store_type != DEFAULT_VECTOR_STORE_TYPE:
-            return None
-
-        source = merged_config.get(VECTOR_STORE_PROPERTY, {}).get(SOURCE_PROPERTY)
-        if not source or not os.path.exists(source) or not os.path.isdir(source):
+        if (
+            store_type != DEFAULT_VECTOR_STORE_TYPE
+            or not source
+            or not os.path.exists(source)
+            or not os.path.isdir(source)
+        ):
             return None
 
         docs = FAISS_Store.load_documents(source)
@@ -882,21 +918,28 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
     @classmethod
     def fingerprint_addon(cls, config: Dict[str, Any]) -> Optional[str]:
         """Add a fingerprint of enterprise search policy for the graph."""
-        local_knowledge_data = cls._get_local_knowledge_data(config)
+        parsed_config = EnterpriseSearchPolicyConfig.from_dict(config)
 
+        # Resolve the prompt template
+        default_prompt_template = cls._select_default_prompt_template_based_on_features(
+            parsed_config.check_relevancy, parsed_config.enable_citation
+        )
         prompt_template = get_prompt_template(
-            config.get(PROMPT_CONFIG_KEY),
-            DEFAULT_ENTERPRISE_SEARCH_PROMPT_TEMPLATE,
+            jinja_file_path=parsed_config.prompt_template,
+            default_prompt_template=default_prompt_template,
             log_source_component=EnterpriseSearchPolicy.__name__,
             log_source_method=LOG_COMPONENT_SOURCE_METHOD_FINGERPRINT_ADDON,
         )
 
-        llm_config = resolve_model_client_config(
-            config.get(LLM_CONFIG_KEY), EnterpriseSearchPolicy.__name__
-        )
-        embedding_config = resolve_model_client_config(
-            config.get(EMBEDDINGS_CONFIG_KEY), EnterpriseSearchPolicy.__name__
+        # Fetch the local knowledge data in case FAISS is used
+        local_knowledge_data = cls._get_local_knowledge_data(
+            parsed_config.vector_store_type, parsed_config.vector_store_source
         )
+
+        # Get the resolved LLM and embeddings configurations
+        llm_config = parsed_config.llm_config
+        embedding_config = parsed_config.embeddings_config
+
         return deep_container_fingerprint(
             [prompt_template, local_knowledge_data, llm_config, embedding_config]
         )
@@ -922,7 +965,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         Returns:
             The post-processed LLM answer.
         """
-        logger.debug(
+        structlogger.debug(
             "enterprise_search_policy.post_process_citations", llm_answer=llm_answer
         )
 
@@ -1085,24 +1128,97 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
     @classmethod
     def _perform_health_checks(
-        cls, config: Dict[Text, Any], log_source_method: str
+        cls,
+        llm_config: Dict[Text, Any],
+        embeddings_config: Dict[Text, Any],
+        log_source_method: str,
     ) -> None:
-        # Perform health check of the LLM client config
-        llm_config = resolve_model_client_config(config.get(LLM_CONFIG_KEY, {}))
+        """
+        Perform the health checks using resolved LLM and embeddings configurations.
+        Resolved means the configuration is either:
+        - A reference to a model group that has already been expanded into
+          its corresponding configuration using the information from
+          `endpoints.yml`, or
+        - A full configuration for the embedder defined directly (i.e. not
+          relying on model groups or indirections).
+
+        Args:
+            llm_config: A resolved LLM configuration.
+            embeddings_config: A resolved embeddings configuration.
+            log_source_method: The method health checks has been called from.
+
+        """
         cls.perform_llm_health_check(
             llm_config,
             DEFAULT_LLM_CONFIG,
             log_source_method,
             EnterpriseSearchPolicy.__name__,
         )
-
-        # Perform health check of the embeddings client config
-        embeddings_config = resolve_model_client_config(
-            config.get(EMBEDDINGS_CONFIG_KEY, {})
-        )
         cls.perform_embeddings_health_check(
             embeddings_config,
             DEFAULT_EMBEDDINGS_CONFIG,
             log_source_method,
             EnterpriseSearchPolicy.__name__,
         )
+
+    @classmethod
+    def get_system_default_prompt_based_on_config(cls, config: Dict[str, Any]) -> str:
+        """
+        Resolves the default prompt template for Enterprise Search Policy based on
+        the component's configuration.
+
+        - The old prompt is selected when both citation and relevancy check are either
+          disabled or not set in the configuration.
+        - The citation prompt is used when citation is enabled and relevancy check is
+          either disabled or not set in the configuration.
+        - The relevancy check prompt is only used when relevancy check is enabled.
+
+        Args:
+            config: The component's configuration.
+
+        Returns:
+            The resolved jinja prompt template as a string.
+        """
+        # Get the feature flags
+        parsed_config = EnterpriseSearchPolicyConfig.from_dict(config)
+        # Based on the enabled features (citation, relevancy check) fetch the
+        # appropriate default prompt
+        default_prompt = cls._select_default_prompt_template_based_on_features(
+            parsed_config.check_relevancy, parsed_config.enable_citation
+        )
+
+        return default_prompt
+
+    @classmethod
+    def _select_default_prompt_template_based_on_features(
+        cls,
+        relevancy_check_enabled: bool,
+        citation_enabled: bool,
+    ) -> str:
+        """
+        Returns the appropriate default prompt template based on the feature flags.
+
+        The selection follows this priority:
+        1. If relevancy check is enabled, return the prompt that includes both relevancy
+           and citation blocks.
+        2. If only citation is enabled, return the prompt with citation blocks.
+        3. Otherwise, fall back to the legacy default prompt template.
+
+        Args:
+            relevancy_check_enabled: Whether the LLM-generated answer should undergo
+                relevancy evaluation.
+            citation_enabled: Whether citations should be included in the generated
+                answer.
+
+        Returns:
+            The default prompt template corresponding to the enabled features.
+        """
+        if relevancy_check_enabled:
+            # ES prompt that has relevancy check and citations blocks
+            return DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_RELEVANCY_CHECK_AND_CITATION_TEMPLATE  # noqa: E501
+        elif citation_enabled:
+            # ES prompt with citation's block - backward compatibility
+            return DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_CITATION_TEMPLATE
+        else:
+            # Legacy ES prompt - backward compatibility
+            return DEFAULT_ENTERPRISE_SEARCH_PROMPT_TEMPLATE