unique_toolkit 0.8.15__py3-none-any.whl → 0.8.16__py3-none-any.whl

unique_toolkit/evals/hallucination/service.py

@@ -0,0 +1,57 @@
+import logging
+
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.schemas import EvaluationMetricInput, EvaluationMetricResult
+
+
+from .constants import hallucination_metric_default_config
+from .utils import check_hallucination
+
+SYSTEM_MSG_KEY = "systemPrompt"
+USER_MSG_KEY = "userPrompt"
+SYSTEM_MSG_DEFAULT_KEY = "systemPromptDefault"
+USER_MSG_DEFAULT_KEY = "userPromptDefault"
+
+
+class HallucinationEvaluator:
+    def __init__(self, event: ChatEvent):
+        self.event = event
+
+        self.logger = logging.getLogger(f"HallucinationEvaluator.{__name__}")
+
+    async def analyze(
+        self,
+        input: EvaluationMetricInput,
+        config: EvaluationMetricConfig = hallucination_metric_default_config,
+    ) -> EvaluationMetricResult | None:
+        """
+        Analyzes the level of hallucination in the generated output by comparing it with the input
+        and the provided contexts or history. The analysis classifies the hallucination level as:
+        - low
+        - medium
+        - high
+
+        If no contexts or history are referenced in the generated output, the method verifies
+        that the output does not contain any relevant information to answer the question.
+
+        This method calls `check_hallucination` to perform the actual analysis. The `check_hallucination`
+        function handles the evaluation using the company ID from the event, the provided input, and the configuration.
+
+        Args:
+            input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+            config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
+
+        Returns:
+            EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the analysis cannot be performed.
+
+        Raises:
+            EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
+        """
+        if config.enabled is False:
+            self.logger.info("Hallucination metric is not enabled.")
+            return None
+
+        return await check_hallucination(
+            company_id=self.event.company_id, input=input, config=config
+        )

unique_toolkit/evals/hallucination/utils.py

@@ -0,0 +1,213 @@
+import logging
+from string import Template
+
+from unique_toolkit.content.schemas import ContentChunk
+from unique_toolkit.language_model.schemas import (
+    LanguageModelMessages,
+    LanguageModelStreamResponse,
+    LanguageModelSystemMessage,
+    LanguageModelUserMessage,
+)
+from unique_toolkit.language_model.service import LanguageModelService
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.output_parser import parse_eval_metric_result
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+from .constants import (
+    SYSTEM_MSG_DEFAULT_KEY,
+    SYSTEM_MSG_KEY,
+    USER_MSG_DEFAULT_KEY,
+    USER_MSG_KEY,
+    hallucination_required_input_fields,
+)
+from .prompts import (
+    HALLUCINATION_METRIC_SYSTEM_MSG,
+    HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    HALLUCINATION_METRIC_USER_MSG,
+    HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+)
+
+
+async def check_hallucination(
+    company_id: str,
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+) -> EvaluationMetricResult:
+    """
+    Analyzes the level of hallucination in the generated output by comparing it with the provided input
+    and the contexts or history. The analysis classifies the hallucination level as:
+    - low
+    - medium
+    - high
+
+    If no contexts or history are referenced in the generated output, the method checks that the output
+    does not contain any relevant information to answer the question.
+
+    This method performs the following steps:
+    1. Checks if the hallucination metric is enabled using the provided `config`.
+    2. Logs the start of the analysis using the provided `logger`.
+    3. Validates the required fields in the `input` data.
+    4. Retrieves the messages using the `_get_msgs` method.
+    5. Calls `LanguageModelService.complete_async_util` to get a completion result.
+    6. Parses and returns the evaluation metric result based on the content of the completion result.
+
+    Args:
+        company_id (str): The company ID for the analysis.
+        input (EvaluationMetricInput): The input data used for evaluation, including the generated output and reference information.
+        config (EvaluationMetricConfig, optional): Configuration settings for the evaluation. Defaults to `hallucination_metric_default_config`.
+        logger (Optional[logging.Logger], optional): The logger used for logging information and errors. Defaults to the logger for the current module.
+
+    Returns:
+        EvaluationMetricResult | None: The result of the evaluation, indicating the level of hallucination. Returns `None` if the metric is not enabled or if an error occurs.
+
+    Raises:
+        EvaluatorException: If the context texts are empty, required fields are missing, or an error occurs during the evaluation.
+    """
+
+    logger = logging.getLogger(f"check_hallucination.{__name__}")
+
+    model_name = config.language_model.name
+    logger.info(f"Analyzing level of hallucination with {model_name}.")
+
+    input.validate_required_fields(hallucination_required_input_fields)
+
+    try:
+        msgs = _get_msgs(input, config, logger)
+        result = await LanguageModelService.complete_async_util(
+            company_id=company_id, messages=msgs, model_name=model_name
+        )
+        result_content = result.choices[0].message.content
+        if not result_content:
+            error_message = "Hallucination evaluation did not return a result."
+            raise EvaluatorException(
+                error_message=error_message,
+                user_message=error_message,
+            )
+        return parse_eval_metric_result(
+            result_content,  # type: ignore
+            EvaluationMetricName.HALLUCINATION,
+        )
+    except Exception as e:
+        error_message = "Error occurred during hallucination metric analysis"
+        raise EvaluatorException(
+            error_message=f"{error_message}: {e}",
+            user_message=error_message,
+            exception=e,
+        )
+
+
+def _get_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+    logger: logging.Logger,
+):
+    """
+    Composes the messages for hallucination analysis based on the provided input and configuration.
+
+    This method decides how to compose the messages based on the availability of context texts and history
+    message texts in the `input`
+
+    Args:
+        input (EvaluationMetricInput): The input data that includes context texts and history message texts
+                                      for the analysis.
+        config (EvaluationMetricConfig): The configuration settings for composing messages.
+        logger (Optional[logging.Logger], optional): The logger used for logging debug information.
+                                                     Defaults to the logger for the current module.
+
+    Returns:
+        The composed messages as per the provided input and configuration. The exact type and structure
+        depend on the implementation of the `compose_msgs` and `compose_msgs_default` methods.
+
+    """
+    if input.context_texts or input.history_messages:
+        logger.debug("Using context / history for hallucination evaluation.")
+        return _compose_msgs(input, config)
+    else:
+        logger.debug("No contexts and history provided for hallucination evaluation.")
+        return _compose_msgs_default(input, config)
+
+
+def _compose_msgs(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the hallucination analysis messages.
+    """
+    system_msg_content = _get_system_prompt_with_contexts(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt_with_contexts(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text,
+        contexts_text=input.get_joined_context_texts(tag_name="reference"),
+        history_messages_text=input.get_joined_history_texts(tag_name="conversation"),
+        output_text=input.output_text,
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _compose_msgs_default(
+    input: EvaluationMetricInput,
+    config: EvaluationMetricConfig,
+):
+    """
+    Composes the hallucination analysis prompt without messages.
+    """
+    system_msg_content = _get_system_prompt_default(config)
+    system_msg = LanguageModelSystemMessage(content=system_msg_content)
+
+    user_msg_templ = Template(_get_user_prompt_default(config))
+    user_msg_content = user_msg_templ.substitute(
+        input_text=input.input_text,
+        output_text=input.output_text,
+    )
+    user_msg = LanguageModelUserMessage(content=user_msg_content)
+    return LanguageModelMessages([system_msg, user_msg])
+
+
+def _get_system_prompt_with_contexts(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_KEY,
+        HALLUCINATION_METRIC_SYSTEM_MSG,
+    )
+
+
+def _get_user_prompt_with_contexts(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_KEY,
+        HALLUCINATION_METRIC_USER_MSG,
+    )
+
+
+def _get_system_prompt_default(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        SYSTEM_MSG_DEFAULT_KEY,
+        HALLUCINATION_METRIC_SYSTEM_MSG_DEFAULT,
+    )
+
+
+def _get_user_prompt_default(config: EvaluationMetricConfig):
+    return config.custom_prompts.setdefault(
+        USER_MSG_DEFAULT_KEY,
+        HALLUCINATION_METRIC_USER_MSG_DEFAULT,
+    )
+
+
+def context_text_from_stream_response(
+    response: LanguageModelStreamResponse, selected_chunks: list[ContentChunk]
+):
+    response_references = response.message.references
+    reference_ids = [reference.source_id for reference in response_references]
+    filtered_contexts: list[str] = []
+    for chunk in selected_chunks:
+        if f"{chunk.id}_{chunk.chunk_id}" in reference_ids:
+            filtered_contexts.append(chunk.text)
+    return filtered_contexts

unique_toolkit/evals/output_parser.py

@@ -0,0 +1,48 @@
+from unique_toolkit.language_model.utils import convert_string_to_json
+from unique_toolkit.evals.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+def parse_eval_metric_result(
+    result: str,
+    metric_name: EvaluationMetricName,
+):
+    """
+    Parses the evaluation metric result.
+    """
+
+    try:
+        parsed_result = convert_string_to_json(result)
+    except Exception as e:
+        error_message = "Error occurred during parsing the evaluation metric result"
+        raise EvaluatorException(
+            user_message=f"{error_message}.",
+            error_message=f"{error_message}: {str(e)}",
+        )
+
+    return EvaluationMetricResult(
+        name=metric_name,
+        value=parsed_result.get("value", "None"),
+        reason=parsed_result.get("reason", "None"),
+    )
+
+
+def parse_eval_metric_result_structured_output(
+    result: EvaluationSchemaStructuredOutput,
+    metric_name: EvaluationMetricName,
+) -> EvaluationMetricResult:
+    """
+    Parses the evaluation metric result.
+    """
+    return EvaluationMetricResult(
+        name=metric_name,
+        value=result.value,
+        reason=result.reason,
+        fact_list=[item.fact for item in result.fact_list],
+    )

unique_toolkit/evals/tests/test_context_relevancy_service.py

@@ -0,0 +1,252 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+from unique_toolkit.app.schemas import ChatEvent
+from unique_toolkit.chat.service import LanguageModelName
+from unique_toolkit.language_model.infos import (
+    LanguageModelInfo,
+)
+from unique_toolkit.language_model.schemas import (
+    LanguageModelAssistantMessage,
+    LanguageModelCompletionChoice,
+    LanguageModelMessages,
+)
+from unique_toolkit.language_model.service import LanguageModelResponse
+from unique_toolkit.evals.config import EvaluationMetricConfig
+from unique_toolkit.evals.context_relevancy.prompts import (
+    CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG,
+)
+from unique_toolkit.evals.context_relevancy.schema import (
+    EvaluationSchemaStructuredOutput,
+)
+from unique_toolkit.evals.context_relevancy.service import (
+    ContextRelevancyEvaluator,
+)
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.schemas import (
+    EvaluationMetricInput,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+
+
+@pytest.fixture
+def event():
+    event = MagicMock(spec=ChatEvent)
+    event.payload = MagicMock()
+    event.payload.user_message = MagicMock()
+    event.payload.user_message.text = "Test query"
+    event.user_id = "user_0"
+    event.company_id = "company_0"
+    return event
+
+
+@pytest.fixture
+def evaluator(event):
+    return ContextRelevancyEvaluator(event)
+
+
+@pytest.fixture
+def basic_config():
+    return EvaluationMetricConfig(
+        enabled=True,
+        name=EvaluationMetricName.CONTEXT_RELEVANCY,
+        language_model=LanguageModelInfo.from_name(
+            LanguageModelName.AZURE_GPT_4o_2024_0806
+        ),
+    )
+
+
+@pytest.fixture
+def structured_config(basic_config):
+    model_info = LanguageModelInfo.from_name(LanguageModelName.AZURE_GPT_4o_2024_0806)
+    return EvaluationMetricConfig(
+        enabled=True,
+        name=EvaluationMetricName.CONTEXT_RELEVANCY,
+        language_model=model_info,
+    )
+
+
+@pytest.fixture
+def sample_input():
+    return EvaluationMetricInput(
+        input_text="test query",
+        context_texts=["test context 1", "test context 2"],
+    )
+
+
+@pytest.mark.asyncio
+async def test_analyze_disabled(evaluator, sample_input, basic_config):
+    basic_config.enabled = False
+    result = await evaluator.analyze(sample_input, basic_config)
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_analyze_empty_context(evaluator, basic_config):
+    input_with_empty_context = EvaluationMetricInput(
+        input_text="test query", context_texts=[]
+    )
+
+    with pytest.raises(EvaluatorException) as exc_info:
+        await evaluator.analyze(input_with_empty_context, basic_config)
+
+    assert "No context texts provided." in str(exc_info.value)
+
+
+@pytest.mark.asyncio
+async def test_analyze_regular_output(evaluator, sample_input, basic_config):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(
+                    content="""{
+                        "value": "high",
+                        "reason": "Test reason"
+                    }"""
+                ),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ) as mock_complete:
+        result = await evaluator.analyze(sample_input, basic_config)
+
+        assert isinstance(result, EvaluationMetricResult)
+        assert result.value.lower() == "high"
+        mock_complete.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_analyze_structured_output(evaluator, sample_input, structured_config):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(
+                    content="HIGH",
+                    parsed={"value": "high", "reason": "Test reason"},
+                ),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    structured_output_schema = EvaluationSchemaStructuredOutput
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ) as mock_complete:
+        result = await evaluator.analyze(
+            sample_input, structured_config, structured_output_schema
+        )
+        assert isinstance(result, EvaluationMetricResult)
+        assert result.value.lower() == "high"
+        mock_complete.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_analyze_structured_output_validation_error(
+    evaluator, sample_input, structured_config
+):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(
+                    content="HIGH", parsed={"invalid": "data"}
+                ),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    structured_output_schema = EvaluationSchemaStructuredOutput
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ):
+        with pytest.raises(EvaluatorException) as exc_info:
+            await evaluator.analyze(
+                sample_input, structured_config, structured_output_schema
+            )
+        assert "Error occurred during structured output validation" in str(
+            exc_info.value
+        )
+
+
+@pytest.mark.asyncio
+async def test_analyze_regular_output_empty_response(
+    evaluator, sample_input, basic_config
+):
+    mock_result = LanguageModelResponse(
+        choices=[
+            LanguageModelCompletionChoice(
+                index=0,
+                message=LanguageModelAssistantMessage(content=""),
+                finish_reason="stop",
+            )
+        ]
+    )
+
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        return_value=mock_result,
+    ):
+        with pytest.raises(EvaluatorException) as exc_info:
+            await evaluator.analyze(sample_input, basic_config)
+        assert "did not return a result" in str(exc_info.value)
+
+
+def test_compose_msgs_regular(evaluator, sample_input, basic_config):
+    messages = evaluator._compose_msgs(
+        sample_input, basic_config, enable_structured_output=False
+    )
+
+    assert isinstance(messages, LanguageModelMessages)
+    assert messages.root[0].content == CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG
+    assert isinstance(messages.root[1].content, str)
+    assert "test query" in messages.root[1].content
+    assert "test context 1" in messages.root[1].content
+    assert "test context 2" in messages.root[1].content
+
+
+def test_compose_msgs_structured(evaluator, sample_input, structured_config):
+    messages = evaluator._compose_msgs(
+        sample_input, structured_config, enable_structured_output=True
+    )
+
+    assert isinstance(messages, LanguageModelMessages)
+    assert len(messages.root) == 2
+    assert (
+        messages.root[0].content != CONTEXT_RELEVANCY_METRIC_SYSTEM_MSG
+    )  # Should use structured output prompt
+    assert isinstance(messages.root[1].content, str)
+    assert "test query" in messages.root[1].content
+    assert "test context 1" in messages.root[1].content
+    assert "test context 2" in messages.root[1].content
+
+
+@pytest.mark.asyncio
+async def test_analyze_unknown_error(evaluator, sample_input, basic_config):
+    with patch.object(
+        evaluator.language_model_service,
+        "complete_async",
+        side_effect=Exception("Unknown error"),
+    ):
+        with pytest.raises(EvaluatorException) as exc_info:
+            await evaluator.analyze(sample_input, basic_config)
+        assert "Unknown error occurred during context relevancy metric analysis" in str(
+            exc_info.value
+        )

unique_toolkit/evals/tests/test_output_parser.py

@@ -0,0 +1,80 @@
+import pytest
+
+from unique_toolkit.evals.context_relevancy.schema import EvaluationSchemaStructuredOutput, Fact
+from unique_toolkit.evals.exception import EvaluatorException
+from unique_toolkit.evals.output_parser import parse_eval_metric_result, parse_eval_metric_result_structured_output
+from unique_toolkit.evals.schemas import EvaluationMetricName, EvaluationMetricResult
+
+
+
+
+def test_parse_eval_metric_result_success():
+    # Test successful parsing with all fields
+    result = '{"value": "high", "reason": "Test reason"}'
+    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "Test reason"
+    assert parsed.fact_list == []
+
+
+def test_parse_eval_metric_result_missing_fields():
+    # Test parsing with missing fields (should use default "None")
+    result = '{"value": "high"}'
+    parsed = parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "None"
+    assert parsed.fact_list == []
+
+
+def test_parse_eval_metric_result_invalid_json():
+    # Test parsing with invalid JSON
+    result = "invalid json"
+    with pytest.raises(EvaluatorException) as exc_info:
+        parse_eval_metric_result(result, EvaluationMetricName.CONTEXT_RELEVANCY)
+
+    assert "Error occurred during parsing the evaluation metric result" in str(
+        exc_info.value
+    )
+
+
+def test_parse_eval_metric_result_structured_output_basic():
+    # Test basic structured output without fact list
+    result = EvaluationSchemaStructuredOutput(value="high", reason="Test reason")
+    parsed = parse_eval_metric_result_structured_output(
+        result, EvaluationMetricName.CONTEXT_RELEVANCY
+    )
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "Test reason"
+    assert parsed.fact_list == []
+
+
+def test_parse_eval_metric_result_structured_output_with_facts():
+    # Test structured output with fact list
+    result = EvaluationSchemaStructuredOutput(
+        value="high",
+        reason="Test reason",
+        fact_list=[
+            Fact(fact="Fact 1"),
+            Fact(fact="Fact 2"),
+        ],
+    )
+    parsed = parse_eval_metric_result_structured_output(
+        result, EvaluationMetricName.CONTEXT_RELEVANCY
+    )
+
+    assert isinstance(parsed, EvaluationMetricResult)
+    assert parsed.name == EvaluationMetricName.CONTEXT_RELEVANCY
+    assert parsed.value == "high"
+    assert parsed.reason == "Test reason"
+    assert parsed.fact_list == ["Fact 1", "Fact 2"]
+    assert isinstance(parsed.fact_list, list)
+    assert len(parsed.fact_list) == 2  # None fact should be filtered out

unique_toolkit/history_manager/history_manager.py

@@ -18,6 +18,7 @@ from unique_toolkit.language_model.schemas import (
     LanguageModelFunction,
     LanguageModelMessage,
     LanguageModelMessageRole,
+    LanguageModelMessages,
     LanguageModelSystemMessage,
     LanguageModelToolMessage,
     LanguageModelUserMessage,
@@ -218,13 +219,7 @@ class HistoryManager:
         rendered_user_message_string: str,
         rendered_system_message_string: str,
         remove_from_text: Callable[[str], Awaitable[str]]
-    ) -> list[
-        LanguageModelMessage
-        | LanguageModelToolMessage
-        | LanguageModelAssistantMessage
-        | LanguageModelSystemMessage
-        | LanguageModelUserMessage
-    ]:
+    ) -> LanguageModelMessages:
         messages = await self._token_reducer.get_history_for_model_call(
             original_user_message=original_user_message,
             rendered_user_message_string=rendered_user_message_string,
@@ -232,4 +227,4 @@ class HistoryManager:
             loop_history=self._loop_history,
             remove_from_text=remove_from_text,
         )
-        return messages.root
+        return messages

unique_toolkit/language_model/schemas.py

@@ -129,6 +129,14 @@ class LanguageModelStreamResponse(BaseModel):
     message: LanguageModelStreamResponseMessage
     tool_calls: list[LanguageModelFunction] | None = None
 
+    def is_empty(self) -> bool:
+        """
+        Check if the stream response is empty.
+        An empty stream response has no text and no tool calls.
+        """
+        return not self.message.original_text and not self.tool_calls
+
+
     def to_openai_param(self) -> ChatCompletionAssistantMessageParam:
         return ChatCompletionAssistantMessageParam(
             role="assistant",