unique_toolkit 0.8.14__py3-none-any.whl → 0.8.16__py3-none-any.whl

unique_toolkit/evals/hallucination/constants.py

@@ -0,0 +1,61 @@
+from typing import Any
+
+from pydantic import Field
+
+from unique_toolkit._common.validators import LMI
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.hallucination.prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricInputFieldName,
+    EvaluationMetricName,
+)
+from unique_toolkit.language_model.infos import LanguageModelInfo, LanguageModelName
+
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
+
+
+class HallucinationConfig(EvaluationMetricConfig):
+    enabled: bool = False
+    name: EvaluationMetricName = EvaluationMetricName.HALLUCINATION
+    language_model: LMI = LanguageModelInfo.from_name(
+        LanguageModelName.AZURE_GPT_35_TURBO_0125,
+    )
+    additional_llm_options: dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to the language model.",
+    )
+    custom_prompts: dict = {
+        SYSTEM_MSG_KEY: HALLUCINATION_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: HALLUCINATION_METRIC_USER_MSG,
+        SYSTEM_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+        USER_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    }
+    score_to_label: dict = {
+        "LOW": "GREEN",
+        "MEDIUM": "YELLOW",
+        "HIGH": "RED",
+    }
+    score_to_title: dict = {
+        "LOW": "No Hallucination Detected",
+        "MEDIUM": "Hallucination Warning",
+        "HIGH": "High Hallucination",
+    }
+
+
+hallucination_metric_default_config = HallucinationConfig()
+
+hallucination_required_input_fields = [
+    EvaluationMetricInputFieldName.INPUT_TEXT,
+    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
+    EvaluationMetricInputFieldName.HISTORY_MESSAGES,
+    EvaluationMetricInputFieldName.OUTPUT_TEXT,
+]

unique_toolkit/evals/hallucination/hallucination_evaluation.py

@@ -0,0 +1,92 @@
+from typing import Any
+
+from unique_toolkit.unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.unique_toolkit.evals.evaluation_manager import Evaluation
+from unique_toolkit.unique_toolkit.evals.hallucination.utils import check_hallucination
+from unique_toolkit.unique_toolkit.evals.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.unique_toolkit.evals.hallucination.constants import (
+    HallucinationConfig,
+)
+from unique_toolkit.unique_toolkit.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+)
+
+
+class HallucinationEvaluation(Evaluation):
+    def __init__(
+        self,
+        config: HallucinationConfig,
+        event: ChatEvent,
+        reference_manager: ReferenceManager,
+    ):
+        self.config = config
+        self._company_id = event.company_id
+        self._user_id = event.user_id
+        self._reference_manager = reference_manager
+        self._user_message = event.payload.user_message.text
+        super().__init__(EvaluationMetricName.HALLUCINATION)
+
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:  # type: ignore
+        chunks = self._reference_manager.get_chunks()
+
+        evaluation_result: EvaluationMetricResult = await check_hallucination(
+            company_id=self._company_id,
+            input=EvaluationMetricInput(
+                input_text=self._user_message,
+                context_texts=[context.text for context in chunks],
+                history_messages=[],  # TODO include loop_history messages
+                output_text=loop_response.message.text,
+            ),
+            config=self.config,
+        )
+
+        score_to_label = self.config.score_to_label
+        evaluation_result.is_positive = (
+            score_to_label.get(evaluation_result.value.upper(), "RED") != "RED"
+        )
+        return evaluation_result
+
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        return ChatMessageAssessmentType.HALLUCINATION
+
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        title = self.config.score_to_title.get(
+            evaluation_result.value.upper(), evaluation_result.value
+        )
+        label = ChatMessageAssessmentLabel(
+            self.config.score_to_label.get(
+                evaluation_result.value.upper(), evaluation_result.value.upper()
+            )
+        )
+        status = (
+            ChatMessageAssessmentStatus.DONE
+            if not evaluation_result.error
+            else ChatMessageAssessmentStatus.ERROR
+        )
+
+        return EvaluationAssessmentMessage(
+            status=status,
+            title=title,
+            explanation=evaluation_result.reason,
+            label=label,
+            type=self.get_assessment_type(),
+        )

unique_toolkit/evals/hallucination/prompts.py

@@ -0,0 +1,79 @@
+HALLUCINATION_METRIC_SYSTEM_MSG = """
+You will receive a question, references, a conversation between a user and an agent, and an output. 
+The output is the answer to the question. 
+Your task is to evaluate if the output is fully supported by the information provided in the references and conversation, and provide explanations on your judgement in 2 sentences.
+
+Use the following entailment scale to generate a score:
+[low] - All information in output is supported by the references/conversation, or extractions from the references/conversation.
+[medium] - The output is supported by the references/conversation to some extent, but there is at least some information in the output that is not discussed in the references/conversation. For example, if an instruction asks about two concepts and the references/conversation only discusses either of them, it should be considered a [medium] hallucination level.
+[high] - The output contains information that is not part of the references/conversation, is unrelated to the references/conversation, or contradicts the references/conversation.
+
+Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check whether the output is supported by the references/conversation, and not whether the output is correct or not. Also do not evaluate if the references/conversation contain further information that is not part of the output but could be relevant to the question. If the output mentions a plot or chart, ignore this information in your evaluation.
+
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your judgement of the evaluation,
+ "value": decision, must be one of the following: ["high", "medium", "low"]
+}                                                  
+"""
+
+HALLUCINATION_METRIC_USER_MSG = """
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+References:
+'''
+$contexts_text
+'''
+
+Conversation:
+'''
+$history_messages_text
+'''
+
+Output:
+'''
+$output_text
+'''
+
+Answer as JSON:
+"""
+
+HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT = """
+You will receive a question and an output. 
+The output is the answer to the question. 
+The situation is that no references could be found to answer the question. Your task is to evaluate if the output contains any information to answer the question,
+and provide a short explanations of your reasoning in 2 sentences. Also mention in your explanation that no references were provided to answer the question.
+
+Use the following entailment scale to generate a score:
+[low] - The output does not contain any information to answer the question.
+[medium] - The output contains some information to answer the question, but does not answer the question entirely. 
+[high] - The output answers the question.
+
+It is not considered an answer when the output relates to the questions subject. Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check that the output does not answer the question, and not whether the output is correct or not.
+Your answer must be in JSON format:
+{
+ "reason": Your explanation of your reasoning of the evaluation,
+ "value": decision, must be one of the following: ["low", "medium", "high"]
+}
+"""
+
+HALLUCINATION_METRIC_USER_MSG_DEFAULT = """                                                  
+Here is the data:
+
+Input:
+'''
+$input_text
+'''
+
+Output:
+'''
+$output_text
+'''
+
+Answer as JSON:
+"""

unique_toolkit/evals/hallucination/service.py

@@ -0,0 +1,57 @@
+import logging
+
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.schemas import EvaluationMetricInput, EvaluationMetricResult
+
+
+from .constants import hallucination_metric_default_config
+from .utils import check_hallucination
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
+
+
+class HallucinationEvaluator:
+    def __init__(self, event: ChatEvent):
+        self.event = event
+
+        self.logger = logging.getLogger(f"HallucinationEvaluator.{__name__}")
+
+    async def analyze(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig = hallucination_metric_default_config,
+    ) -> EvaluationMetricResult | None:
+        """
+        Analyzes the level of hallucination in the generated output by comparing it with the input
+        and the provided contexts or history. The analysis classifies the hallucination level as:
+        - low
+        - medium
+        - high
+
+        If no contexts or history are referenced in the generated output, the method verifies
+        that the output does not contain any relevant information to answer the question.
+
+        This method calls `check_hallucination` to perform the actual analysis. The `check_hallucination`
+        function handles the evaluation using the company ID from the event, the provided input, and the configuration.
+
+        Args:
+            input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+            config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
+
+        Returns:
+            EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the analysis cannot be performed.
+
+        Raises:
+            EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
+        """
+        if config.enabled is False:
+            self.logger.info("Hallucination metric is not enabled.")
+            return None
+
+        return await check_hallucination(
+            company_id=self.event.company_id, input=input, config=config
+        )

unique_toolkit/evals/hallucination/utils.py

@@ -0,0 +1,213 @@
+import logging
+from string import Template
+
+from unique_toolkit.content.schemas import ContentChunk
+from unique_toolkit.language_model.schemas import (
+    LanguageModelMessages,
+    LanguageModelStreamResponse,
+    LanguageModelSystemMessage,
+    LanguageModelUserMessage,
+)
+from unique_toolkit.language_model.service import LanguageModelService
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.output_parser import parse_eval_metric_result
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+from .constants import (
+    SYSTEM_MSG_DEFAULT_KEY,
+    SYSTEM_MSG_KEY,
+    USER_MSG_DEFAULT_KEY,
+    USER_MSG_KEY,
+    hallucination_required_input_fields,
+)
+from .prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+
+
+async def check_hallucination(
+    company_id: str,
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+) -> EvaluationMetricResult:
+    """
+    Analyzes the level of hallucination in the generated output by comparing it with the provided input
+    and the contexts or history. The analysis classifies the hallucination level as:
+    - low
+    - medium
+    - high
+
+    If no contexts or history are referenced in the generated output, the method checks that the output
+    does not contain any relevant information to answer the question.
+
+    This method performs the following steps:
+    1. Checks if the hallucination metric is enabled using the provided `config`.
+    2. Logs the start of the analysis using the provided `logger`.
+    3. Validates the required fields in the `input` data.
+    4. Retrieves the messages using the `_get_msgs` method.
+    5. Calls `LanguageModelService.complete_async_util` to get a completion result.
+    6. Parses and returns the evaluation metric result based on the content of the completion result.
+
+    Args:
+        company_id (str): The company ID for the analysis.
+        input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+        config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
+        logger (Optional[logging.Logger], optional): The logger used for logging information and errors. Defaults to the logger for the current module.
+
+    Returns:
+        EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the metric is not enabled or if an error occurs.
+
+    Raises:
+        EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
+    """
+
+    logger = logging.getLogger(f"check_hallucination.{__name__}")
+
+    model_name = config.language_model.name
+    logger.info(f"Analyzing level of hallucination with {model_name}.")
+
+    input.validate_required_fields(hallucination_required_input_fields)
+
+    try:
+        msgs = _get_msgs(input, config, logger)
+        result = await LanguageModelService.complete_async_util(
+            company_id=company_id, messages=msgs, model_name=model_name
+        )
+        result_content = result.choices[0].message.content
+        if not result_content:
+            error_message = "Hallucination evaluation did not return a result."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+            )
+        return parse_eval_metric_result(
+            result_content,  # type: ignore
+            EvaluationMetricName.HALLUCINATION,
+        )
+    except Exception as e:
+        error_message = "Error occurred during hallucination metric analysis"
+        raise EvaluatorException(
+            error_message=f"{error_message}: {e}",
+            user_message=error_message,
+            exception=e,
+        )
+
+
+def _get_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+    logger: logging.Logger,
+):
+    """
+    Composes the messages for hallucination analysis based on the provided input and configuration.
+
+    This method decides how to compose the messages based on the availability of context texts and history
+    message texts in the `input`
+
+    Args:
+        input (EvaluationMetricInput): The input data that includes context texts and history message texts
+                                      for the analysis.
+        config (EvaluationMetricConfig): The configuration settings for composing messages.
+        logger (Optional[logging.Logger], optional): The logger used for logging debug information.
+                                                     Defaults to the logger for the current module.
+
+    Returns:
+        The composed messages as per the provided input and configuration. The exact type and structure
+        depend on the implementation of the `compose_msgs` and `compose_msgs_default` methods.
+
+    """
+    if input.context_texts or input.history_messages:
+        logger.debug("Using context / history for hallucination evaluation.")
+        return _compose_msgs(input, config)
+    else:
+        logger.debug("No contexts and history provided for hallucination evaluation.")
+        return _compose_msgs_default(input, config)
+
+
+def _compose_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the hallucination analysis messages.
+    """
+    system_msg_content = _get_system_prompt_with_contexts(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt_with_contexts(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text,
+        contexts_text=input.get_joined_context_texts(tag_name="reference"),
+        history_messages_text=input.get_joined_history_texts(tag_name="conversation"),
+        output_text=input.output_text,
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _compose_msgs_default(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the hallucination analysis prompt without messages.
+    """
+    system_msg_content = _get_system_prompt_default(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt_default(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text,
+        output_text=input.output_text,
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _get_system_prompt_with_contexts(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_KEY,
+        HALLUCINATION_METRIC_SYSTEM_MSG,
+    )
+
+
+def _get_user_prompt_with_contexts(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_KEY,
+        HALLUCINATION_METRIC_USER_MSG,
+    )
+
+
+def _get_system_prompt_default(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_DEFAULT_KEY,
+        HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    )
+
+
+def _get_user_prompt_default(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_DEFAULT_KEY,
+        HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    )
+
+
+def context_text_from_stream_response(
+    response: LanguageModelStreamResponse, selected_chunks: list[ContentChunk]
+):
+    response_references = response.message.references
+    reference_ids = [reference.source_id for reference in response_references]
+    filtered_contexts: list[str] = []
+    for chunk in selected_chunks:
+        if f"{chunk.id}_{chunk.chunk_id}" in reference_ids:
+            filtered_contexts.append(chunk.text)
+    return filtered_contexts

unique_toolkit/evals/output_parser.py

@@ -0,0 +1,48 @@
+from unique_toolkit.language_model.utils import convert_string_to_json
+from unique_toolkit.evals.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+def parse_eval_metric_result(
+    result: str,
+    metric_name: EvaluationMetricName,
+):
+    """
+    Parses the evaluation metric result.
+    """
+
+    try:
+        parsed_result = convert_string_to_json(result)
+    except Exception as e:
+        error_message = "Error occurred during parsing the evaluation metric result"
+        raise EvaluatorException(
+            user_message=f"{error_message}.",
+            error_message=f"{error_message}: {str(e)}",
+        )
+
+    return EvaluationMetricResult(
+        name=metric_name,
+        value=parsed_result.get("value", "None"),
+        reason=parsed_result.get("reason", "None"),
+    )
+
+
+def parse_eval_metric_result_structured_output(
+    result: EvaluationSchemaStructuredOutput,
+    metric_name: EvaluationMetricName,
+) -> EvaluationMetricResult:
+    """
+    Parses the evaluation metric result.
+    """
+    return EvaluationMetricResult(
+        name=metric_name,
+        value=result.value,
+        reason=result.reason,
+        fact_list=[item.fact for item in result.fact_list],
+    )