unique_toolkit 0.7.7__py3-none-any.whl → 1.23.0__py3-none-any.whl

unique_toolkit/agentic/evaluation/context_relevancy/service.py

@@ -0,0 +1,273 @@
+import logging
+from typing import overload
+
+from pydantic import BaseModel, ValidationError
+from typing_extensions import deprecated
+
+from unique_toolkit._common.validate_required_values import (
+    validate_required_values,
+)
+from unique_toolkit.agentic.evaluation.config import EvaluationMetricConfig
+from unique_toolkit.agentic.evaluation.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
+from unique_toolkit.agentic.evaluation.output_parser import (
+    parse_eval_metric_result,
+    parse_eval_metric_result_structured_output,
+)
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricInputFieldName,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.app.schemas import BaseEvent, ChatEvent
+from unique_toolkit.language_model.default_language_model import DEFAULT_GPT_4o
+from unique_toolkit.language_model.infos import (
+    LanguageModelInfo,
+    ModelCapabilities,
+)
+from unique_toolkit.language_model.prompt import Prompt
+from unique_toolkit.language_model.schemas import (
+    LanguageModelMessages,
+)
+from unique_toolkit.language_model.service import (
+    LanguageModelService,
+)
+
+from .prompts import (
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG_STRUCTURED_OUTPUT,
+    CONTEXT_RELEVANCY_METRIC_USER_MSG,
+    CONTEXT_RELEVANCY_METRIC_USER_MSG_STRUCTURED_OUTPUT,
+)
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+
+default_config = EvaluationMetricConfig(
+    enabled=False,
+    name=EvaluationMetricName.CONTEXT_RELEVANCY,
+    language_model=LanguageModelInfo.from_name(DEFAULT_GPT_4o),
+    custom_prompts={
+        SYSTEM_MSG_KEY: CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: CONTEXT_RELEVANCY_METRIC_USER_MSG,
+    },
+)
+
+relevancy_required_input_fields = [
+    EvaluationMetricInputFieldName.INPUT_TEXT,
+    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
+]
+
+
+class ContextRelevancyEvaluator:
+    @deprecated(
+        "Use __init__ with company_id and user_id instead or use the classmethod `from_event`"
+    )
+    @overload
+    def __init__(self, event: ChatEvent | BaseEvent):
+        """
+        Initialize the ContextRelevancyEvaluator with an event (deprecated)
+        """
+
+    @overload
+    def __init__(self, *, company_id: str, user_id: str):
+        """
+        Initialize the ContextRelevancyEvaluator with a company_id and user_id
+        """
+
+    def __init__(
+        self,
+        event: ChatEvent | BaseEvent | None = None,
+        company_id: str | None = None,
+        user_id: str | None = None,
+    ):
+        if isinstance(event, (ChatEvent, BaseEvent)):
+            self.language_model_service = LanguageModelService.from_event(event)
+        else:
+            [company_id, user_id] = validate_required_values([company_id, user_id])
+            self.language_model_service = LanguageModelService(
+                company_id=company_id, user_id=user_id
+            )
+
+        # Setup the logger
+        module_name = "ContextRelevancyEvaluator"
+        self.logger = logging.getLogger(f"{module_name}.{__name__}")
+
+    @classmethod
+    def from_event(cls, event: ChatEvent | BaseEvent):
+        return cls(company_id=event.company_id, user_id=event.user_id)
+
+    async def analyze(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig = default_config,
+        structured_output_schema: type[BaseModel] | None = None,
+    ) -> EvaluationMetricResult | None:
+        """
+        Analyzes the level of relevancy of a context by comparing
+        it with the input text.
+
+        Args:
+            input (EvaluationMetricInput): The input for the metric.
+            config (EvaluationMetricConfig): The configuration for the metric.
+
+        Returns:
+            EvaluationMetricResult | None
+
+        Raises:
+            EvaluatorException: If the context texts are empty or required fields are missing or error occurred during evaluation.
+        """
+        if config.enabled is False:
+            self.logger.info("Hallucination metric is not enabled.")
+            return None
+
+        input.validate_required_fields(relevancy_required_input_fields)
+
+        # TODO: Was already there in monorepo
+        if len(input.context_texts) == 0:  # type: ignore
+            error_message = "No context texts provided."
+            raise EvaluatorException(
+                user_message=error_message,
+                error_message=error_message,
+            )
+
+        try:
+            # Handle structured output if enabled and supported by the model
+            if (
+                structured_output_schema
+                and ModelCapabilities.STRUCTURED_OUTPUT
+                in config.language_model.capabilities
+            ):
+                return await self._handle_structured_output(
+                    input, config, structured_output_schema
+                )
+
+            # Handle regular output
+            return await self._handle_regular_output(input, config)
+
+        except Exception as e:
+            error_message = (
+                "Unknown error occurred during context relevancy metric analysis"
+            )
+            raise EvaluatorException(
+                error_message=f"{error_message}: {e}",
+                user_message=error_message,
+                exception=e,
+            )
+
+    async def _handle_structured_output(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig,
+        structured_output_schema: type[BaseModel],
+    ) -> EvaluationMetricResult:
+        """Handle the structured output case for context relevancy evaluation."""
+        self.logger.info("Using structured output for context relevancy evaluation.")
+        msgs = self._compose_msgs(input, config, enable_structured_output=True)
+        result = await self.language_model_service.complete_async(
+            messages=msgs,
+            model_name=config.language_model.name,
+            structured_output_model=structured_output_schema,
+            structured_output_enforce_schema=True,
+            other_options=config.additional_llm_options,
+        )
+
+        try:
+            result_content = EvaluationSchemaStructuredOutput.model_validate(
+                result.choices[0].message.parsed
+            )
+        except ValidationError as e:
+            error_message = "Error occurred during structured output validation of the context relevancy evaluation."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+                exception=e,
+            )
+
+        return parse_eval_metric_result_structured_output(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
+
+    async def _handle_regular_output(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig,
+    ) -> EvaluationMetricResult:
+        """Handle the regular output case for context relevancy evaluation."""
+        msgs = self._compose_msgs(input, config, enable_structured_output=False)
+        result = await self.language_model_service.complete_async(
+            messages=msgs,
+            model_name=config.language_model.name,
+            other_options=config.additional_llm_options,
+        )
+
+        result_content = result.choices[0].message.content
+        if not result_content or not isinstance(result_content, str):
+            error_message = "Context relevancy evaluation did not return a result."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+            )
+
+        return parse_eval_metric_result(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
+
+    def _compose_msgs(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig,
+        enable_structured_output: bool,
+    ) -> LanguageModelMessages:
+        """
+        Composes the messages for the relevancy metric.
+        """
+        system_msg_content = self._get_system_prompt(config, enable_structured_output)
+        system_msg = Prompt(system_msg_content).to_system_msg()
+
+        user_msg = Prompt(
+            self._get_user_prompt(config, enable_structured_output),
+            input_text=input.input_text,
+            context_texts=input.get_joined_context_texts(),
+        ).to_user_msg()
+
+        return LanguageModelMessages([system_msg, user_msg])
+
+    def _get_system_prompt(
+        self,
+        config: EvaluationMetricConfig,
+        enable_structured_output: bool,
+    ):
+        if (
+            enable_structured_output
+            and ModelCapabilities.STRUCTURED_OUTPUT
+            in config.language_model.capabilities
+        ):
+            return config.custom_prompts.setdefault(
+                SYSTEM_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG_STRUCTURED_OUTPUT,
+            )
+        else:
+            return config.custom_prompts.setdefault(
+                SYSTEM_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+            )
+
+    def _get_user_prompt(
+        self,
+        config: EvaluationMetricConfig,
+        enable_structured_output: bool,
+    ):
+        if enable_structured_output:
+            return config.custom_prompts.setdefault(
+                USER_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_USER_MSG_STRUCTURED_OUTPUT,
+            )
+        else:
+            return config.custom_prompts.setdefault(
+                USER_MSG_KEY,
+                CONTEXT_RELEVANCY_METRIC_USER_MSG,
+            )

unique_toolkit/agentic/evaluation/evaluation_manager.py

@@ -0,0 +1,218 @@
+import asyncio
+from abc import ABC
+from logging import Logger
+
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.tools.utils.execution.execution import (
+    Result,
+    SafeTaskExecutor,
+)
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.chat.service import ChatService
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+)
+
+
+class Evaluation(ABC):
+    """
+    Abstract base class for evaluation metrics.
+
+    typical use cases include:
+    - Hallucination checking
+    - compliance checking
+    """
+
+    def __init__(self, name: EvaluationMetricName):
+        self.name = name
+
+    def get_name(self) -> EvaluationMetricName:
+        return self.name
+
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        raise NotImplementedError(
+            "Subclasses must implement this method to return the assessment type."
+        )
+
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        raise NotImplementedError(
+            "Subclasses must implement this method to convert evaluation results to assessment messages."
+        )
+
+
+class EvaluationManager:
+    """
+    Manages the evaluation metrics and executes evaluation calls.
+
+    This class is responsible for:
+    - Storing and managing evaluation instances, identified by their unique names.
+    - Executing selected evaluations asynchronously and processing their results.
+    - Tracking the overall success or failure of evaluations.
+    - Integrating with external services like logging and chat systems to display evaluation statuses and results.
+    - Handling errors gracefully, including missing evaluations or failed executions.
+
+    Key Features:
+    - Evaluation Storage: Maintains a dictionary of evaluation instances for quick retrieval.
+    - Asynchronous Execution: Supports concurrent execution of multiple evaluations for efficiency.
+    - Result Processing: Tracks evaluation outcomes and updates the internal state based on results.
+    - Chat Integration: Updates the chat interface with evaluation statuses and detailed assessments.
+    - Error Handling: Provides robust error messages and fallback mechanisms for missing or failed evaluations.
+
+    The EvaluationManager serves as the central hub for managing and executing evaluations.
+    """
+
+    # a hashmap to hold evaluations by their names
+
+    def __init__(
+        self,
+        logger: Logger,
+        chat_service: ChatService,
+    ):
+        self._logger = logger
+        self._chat_service = chat_service
+        self._evaluations: dict[EvaluationMetricName, Evaluation] = {}
+        self._evaluation_passed: bool = True
+
+    def add_evaluation(self, evaluation: Evaluation):
+        self._evaluations[evaluation.get_name()] = evaluation
+
+    def get_evaluation_by_name(self, name: EvaluationMetricName) -> Evaluation | None:
+        return self._evaluations.get(name)
+
+    async def run_evaluations(
+        self,
+        selected_evaluation_names: list[EvaluationMetricName],
+        loop_response: LanguageModelStreamResponse,
+        assistant_message_id: str,
+    ) -> list[EvaluationMetricResult]:
+        task_executor = SafeTaskExecutor(
+            logger=self._logger,
+        )
+
+        tasks = [
+            task_executor.execute_async(
+                self.execute_evaluation_call,
+                loop_response=loop_response,
+                evaluation_name=evaluation_name,
+                assistant_message_id=assistant_message_id,
+            )
+            for evaluation_name in selected_evaluation_names
+        ]
+        evaluation_results = await asyncio.gather(*tasks)
+        evaluation_results_unpacked: list[EvaluationMetricResult] = []
+
+        for i, result in enumerate(evaluation_results):
+            unpacked_evaluation_result = self._create_evaluation_metric_result(
+                result, selected_evaluation_names[i]
+            )
+            if not unpacked_evaluation_result.is_positive:
+                self._evaluation_passed = False
+            evaluation_results_unpacked.append(unpacked_evaluation_result)
+
+        for evaluation_name, evaluation_result in zip(
+            selected_evaluation_names, evaluation_results_unpacked
+        ):
+            evaluation_instance = self.get_evaluation_by_name(evaluation_name)
+            if evaluation_instance:
+                await self._show_message_assessment(
+                    evaluation_instance, evaluation_result, assistant_message_id
+                )
+
+        return evaluation_results_unpacked
+
+    async def execute_evaluation_call(
+        self,
+        evaluation_name: EvaluationMetricName,
+        loop_response: LanguageModelStreamResponse,
+        assistant_message_id: str,
+    ) -> EvaluationMetricResult:
+        self._logger.info(f"Processing tool call: {evaluation_name}")
+
+        evaluation_instance = self.get_evaluation_by_name(evaluation_name)
+
+        if evaluation_instance:
+            # Execute the evaluation
+            await self._create_assistant_message(
+                evaluation_instance, assistant_message_id
+            )
+            evaluation_metric_result: EvaluationMetricResult = (
+                await evaluation_instance.run(loop_response)
+            )
+            return evaluation_metric_result
+
+        return EvaluationMetricResult(
+            name=evaluation_name,
+            is_positive=True,
+            value="RED",
+            reason=f"Evaluation named {evaluation_name} not found",
+            error=Exception("Evaluation named {evaluation_name} not found"),
+        )
+
+    def _create_evaluation_metric_result(
+        self,
+        result: Result[EvaluationMetricResult],
+        evaluation_name: EvaluationMetricName,
+    ) -> EvaluationMetricResult:
+        if not result.success:
+            return EvaluationMetricResult(
+                name=evaluation_name,
+                is_positive=True,
+                value="RED",
+                reason=str(result.exception),
+                error=Exception("Evaluation result is not successful"),
+            )
+        unpacked = result.unpack()
+        if not isinstance(unpacked, EvaluationMetricResult):
+            return EvaluationMetricResult(
+                name=evaluation_name,
+                is_positive=True,
+                value="RED",
+                reason="Evaluation result is not of type EvaluationMetricResult",
+                error=Exception(
+                    "Evaluation result is not of type EvaluationMetricResult"
+                ),
+            )
+        return unpacked
+
+    async def _show_message_assessment(
+        self,
+        evaluation_instance: Evaluation,
+        evaluation_metric_result: EvaluationMetricResult,
+        assistant_message_id: str,
+    ) -> None:
+        evaluation_assessment_message = (
+            await evaluation_instance.evaluation_metric_to_assessment(
+                evaluation_metric_result
+            )
+        )
+        await self._chat_service.modify_message_assessment_async(
+            assistant_message_id=assistant_message_id,
+            status=evaluation_assessment_message.status,
+            title=evaluation_assessment_message.title,
+            explanation=evaluation_assessment_message.explanation,
+            label=evaluation_assessment_message.label,
+            type=evaluation_assessment_message.type,
+        )
+
+    async def _create_assistant_message(
+        self, evaluation_instance: Evaluation, assistant_message_id: str
+    ):
+        await self._chat_service.create_message_assessment_async(
+            assistant_message_id=assistant_message_id,
+            status=ChatMessageAssessmentStatus.PENDING,
+            type=evaluation_instance.get_assessment_type(),
+        )

unique_toolkit/agentic/evaluation/hallucination/constants.py

@@ -0,0 +1,61 @@
+from typing import Any
+
+from pydantic import Field
+
+from unique_toolkit._common.validators import LMI
+from unique_toolkit.agentic.evaluation.config import EvaluationMetricConfig
+from unique_toolkit.agentic.evaluation.hallucination.prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricInputFieldName,
+    EvaluationMetricName,
+)
+from unique_toolkit.language_model.default_language_model import DEFAULT_GPT_4o
+from unique_toolkit.language_model.infos import LanguageModelInfo
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
+
+
+class HallucinationConfig(EvaluationMetricConfig):
+    enabled: bool = False
+    name: EvaluationMetricName = EvaluationMetricName.HALLUCINATION
+    language_model: LMI = LanguageModelInfo.from_name(
+        DEFAULT_GPT_4o,
+    )
+    additional_llm_options: dict[str, Any] = Field(
+        default={},
+        description="Additional options to pass to the language model.",
+    )
+    custom_prompts: dict = {
+        SYSTEM_MSG_KEY: HALLUCINATION_METRIC_SYSTEM_MSG,
+        USER_MSG_KEY: HALLUCINATION_METRIC_USER_MSG,
+        SYSTEM_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+        USER_MSG_DEFAULT_KEY: HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    }
+    score_to_label: dict = {
+        "LOW": "GREEN",
+        "MEDIUM": "YELLOW",
+        "HIGH": "RED",
+    }
+    score_to_title: dict = {
+        "LOW": "No Hallucination Detected",
+        "MEDIUM": "Hallucination Warning",
+        "HIGH": "High Hallucination",
+    }
+
+
+hallucination_metric_default_config = HallucinationConfig()
+
+hallucination_required_input_fields = [
+    EvaluationMetricInputFieldName.INPUT_TEXT,
+    EvaluationMetricInputFieldName.CONTEXT_TEXTS,
+    EvaluationMetricInputFieldName.HISTORY_MESSAGES,
+    EvaluationMetricInputFieldName.OUTPUT_TEXT,
+]

unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py

@@ -0,0 +1,111 @@
+import regex as re
+
+from unique_toolkit.agentic.evaluation.evaluation_manager import Evaluation
+from unique_toolkit.agentic.evaluation.hallucination.constants import (
+    HallucinationConfig,
+)
+from unique_toolkit.agentic.evaluation.hallucination.utils import check_hallucination
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.language_model.reference import _preprocess_message
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+)
+
+
+class HallucinationEvaluation(Evaluation):
+    def __init__(
+        self,
+        config: HallucinationConfig,
+        event: ChatEvent,
+        reference_manager: ReferenceManager,
+    ):
+        self.config = config
+        self._company_id = event.company_id
+        self._user_id = event.user_id
+        self._reference_manager = reference_manager
+        self._user_message = event.payload.user_message.text
+        super().__init__(EvaluationMetricName.HALLUCINATION)
+
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:  # type: ignore
+        all_chunks = self._reference_manager.get_chunks()
+
+        # source numbers from original text
+        ref_pattern = r"\[(\d+)\]"
+        original_text = loop_response.message.original_text
+
+        # preprocess original text to deal with different source patterns
+        original_text_preprocessed = _preprocess_message(original_text)
+
+        source_number_matches = re.findall(ref_pattern, original_text_preprocessed)
+        source_numbers = {int(num) for num in source_number_matches}
+
+        referenced_chunks = [all_chunks[idx] for idx in source_numbers]
+
+        evaluation_result: EvaluationMetricResult = await check_hallucination(
+            company_id=self._company_id,
+            input=EvaluationMetricInput(
+                input_text=self._user_message,
+                context_texts=[context.text for context in referenced_chunks],
+                history_messages=[],  # TODO include loop_history messages
+                output_text=loop_response.message.text,
+            ),
+            config=self.config,
+        )
+
+        score_to_label = self.config.score_to_label
+        evaluation_result.is_positive = (
+            score_to_label.get(evaluation_result.value.upper(), "RED") != "RED"
+        )
+        return evaluation_result
+
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        return ChatMessageAssessmentType.HALLUCINATION
+
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        title = self.config.score_to_title.get(
+            evaluation_result.value.upper(), evaluation_result.value
+        )
+        label = ChatMessageAssessmentLabel(
+            self.config.score_to_label.get(
+                evaluation_result.value.upper(), evaluation_result.value.upper()
+            )
+        )
+        status = (
+            ChatMessageAssessmentStatus.DONE
+            if not evaluation_result.error
+            else ChatMessageAssessmentStatus.ERROR
+        )
+        explanation = evaluation_result.reason
+
+        if status == ChatMessageAssessmentStatus.ERROR:
+            title = "Hallucination Check Error"
+            label = ChatMessageAssessmentLabel.RED
+            explanation = (
+                "An unrecoverable error occurred while evaluating the response."
+            )
+
+        return EvaluationAssessmentMessage(
+            status=status,
+            title=title,
+            explanation=explanation,
+            label=label,
+            type=self.get_assessment_type(),
+        )

unique_toolkit/{evaluators → agentic/evaluation}/hallucination/prompts.py

@@ -8,7 +8,7 @@ Use the following entailment scale to generate a score:
 [medium] - The output is supported by the references/conversation to some extent, but there is at least some information in the output that is not discussed in the references/conversation. For example, if an instruction asks about two concepts and the references/conversation only discusses either of them, it should be considered a [medium] hallucination level.
 [high] - The output contains information that is not part of the references/conversation, is unrelated to the references/conversation, or contradicts the references/conversation.
 
-Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check whether the output is supported by the references/conversation, and not whether the output is correct or not. Also do not evaluate if the references/conversation contain further information that is not part of the output but could be relevant to the qestion.
+Make sure to not use any external information/knowledge to judge whether the output is true or not. Only check whether the output is supported by the references/conversation, and not whether the output is correct or not. Also do not evaluate if the references/conversation contain further information that is not part of the output but could be relevant to the question. If the output mentions a plot or chart, ignore this information in your evaluation.
 
 Your answer must be in JSON format:
 {