unique-toolkit 2026.28.0.dev3__tar.gz → 2026.28.0.dev4__tar.gz

{unique_toolkit-2026.28.0.dev3 → unique_toolkit-2026.28.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 2026.28.0.dev3
+Version: 2026.28.0.dev4
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
 Requires-Dist: openai>=1.105.0,<3
 Requires-Dist: platformdirs>=4.0.0,<5
 Requires-Dist: pillow>=12.1.1
-Requires-Dist: unique-sdk>=2026.28.0.dev4,<2026.28.0rc0
+Requires-Dist: unique-sdk>=2026.28.0.dev5,<2026.28.0rc0
 Requires-Dist: jambo>=0.1.2,<0.2
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: docxtpl>=0.20.1,<0.21

{unique_toolkit-2026.28.0.dev3 → unique_toolkit-2026.28.0.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "2026.28.0.dev3"
+version = "2026.28.0.dev4"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,7 +28,7 @@ dependencies = [
     "openai>=1.105.0,<3",
     "platformdirs>=4.0.0,<5",
     "pillow>=12.1.1",
-    "unique-sdk>=2026.28.0.dev4,<2026.28.0rc0",
+    "unique-sdk>=2026.28.0.dev5,<2026.28.0rc0",
     "jambo>=0.1.2,<0.2",
     "jinja2>=3.1.6,<4",
     "docxtpl>=0.20.1,<0.21",

{unique_toolkit-2026.28.0.dev3 → unique_toolkit-2026.28.0.dev4}/unique_toolkit/agentic/evaluation/hallucination/utils.py

@@ -16,6 +16,8 @@ from unique_toolkit.agentic.evaluation.schemas import (
 )
 from unique_toolkit.content import ContentReference
 from unique_toolkit.content.schemas import ContentChunk
+from unique_toolkit.language_model.builder import MessagesBuilder
+from unique_toolkit.language_model.infos import ModelCapabilities
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessages,
     LanguageModelStreamResponse,
@@ -104,19 +106,28 @@ def _get_msgs(
     """
     Composes the messages for hallucination analysis based on the provided input and configuration.
 
-    This method composes messages with or without context based on the availability of context texts
-    and history message texts in the input.
+    This method composes messages with or without context based on the availability of context texts,
+    history message texts, and (for vision-capable models) context images in the input.
 
     Args:
-        input (EvaluationMetricInput): The input data that includes context texts and history message texts
-                                      for the analysis.
+        input (EvaluationMetricInput): The input data that includes context texts, history message texts,
+                                      and context images for the analysis.
         config (EvaluationMetricConfig): The configuration settings for composing messages.
         logger (Optional[logging.Logger], optional): The logger used for logging debug information.
 
     Returns:
         The composed messages as per the provided input and configuration.
     """
-    has_context = bool(input.context_texts or input.history_messages)
+    # Context images only count as grounding when the model can actually consume
+    # them; otherwise _compose_msgs drops them and the message would be text-only.
+    model_supports_vision = (
+        ModelCapabilities.VISION in config.language_model.capabilities
+    )
+    has_context = bool(
+        input.context_texts
+        or input.history_messages
+        or (input.context_images and model_supports_vision)
+    )
 
     if has_context:
         _LOGGER.debug("Using context / history for hallucination evaluation.")
@@ -165,6 +176,36 @@ def _compose_msgs(
         else None,
         output_text=input.output_text,
     )
+
+    # When images are provided AND the configured model is vision-capable, build
+    # a multimodal user message (rendered user text + one image part per data
+    # URL) so the model can ground the check against the rendered pages. If the
+    # model lacks VISION we drop the images and fall back to the text-only
+    # message: attaching images to a non-vision model would make the LLM call
+    # fail. Without images the message stays the exact text-only form, so
+    # existing callers are unaffected.
+    context_images = input.context_images or []
+    model_supports_vision = (
+        ModelCapabilities.VISION in config.language_model.capabilities
+    )
+    # When has_context is True for an images-only input, the caller already
+    # confirmed the model is vision-capable, so this branch stays unreachable in
+    # that case and only logs for a genuine non-vision misconfiguration.
+    if context_images and not model_supports_vision:
+        _LOGGER.warning(
+            "Hallucination model %s lacks VISION capability; ignoring "
+            "%d context image(s) and grounding on text only.",
+            config.language_model.name,
+            len(context_images),
+        )
+    if context_images and model_supports_vision:
+        return (
+            MessagesBuilder()
+            .append(system_msg)
+            .image_message_append(content=user_msg_content, images=context_images)
+            .build()
+        )
+
     user_msg = LanguageModelUserMessage(content=user_msg_content)
 
     return LanguageModelMessages([system_msg, user_msg])

{unique_toolkit-2026.28.0.dev3 → unique_toolkit-2026.28.0.dev4}/unique_toolkit/agentic/evaluation/schemas.py

@@ -34,6 +34,16 @@ class EvaluationMetricInput(BaseModel):
     context_texts: Optional[list[str]] = None
     history_messages: Optional[list[ChatMessage]] = None
     output_text: Optional[str] = None
+    context_images: Optional[list[str]] = Field(
+        default=None,
+        description=(
+            "Optional base64 data URLs (e.g. 'data:image/png;base64,...') of "
+            "context the metric should evaluate against visually, in addition "
+            "to context_texts. Used by metrics that support a multimodal "
+            "(vision) check; ignored by metrics that do not. Defaults to None "
+            "so existing text-only callers are unaffected."
+        ),
+    )
 
     def get_joined_context_texts(self, tag_name: str = "reference") -> str:
         """

{unique_toolkit-2026.28.0.dev3 → unique_toolkit-2026.28.0.dev4}/unique_toolkit/agentic/evaluation/tests/test_hallucination_utils.py

@@ -20,6 +20,10 @@ from unique_toolkit.agentic.evaluation.schemas import EvaluationMetricInput
 from unique_toolkit.chat.schemas import ChatMessage
 from unique_toolkit.content import ContentReference
 from unique_toolkit.content.schemas import ContentChunk
+from unique_toolkit.language_model.infos import (
+    LanguageModelName,
+    ModelCapabilities,
+)
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessageRole,
     LanguageModelMessages,
@@ -1013,3 +1017,162 @@ def test_compose_msgs__generates_non_empty_messages__from_config_prompts(
     # Assert
     assert len(result.root[0].content) > 0
     assert len(result.root[1].content) > 0
+
+
+_IMAGE_DATA_URL = "data:image/png;base64,QQ=="
+
+
+@pytest.mark.ai
+def test_compose_msgs__builds_multimodal_user_message__with_context_images(
+    hallucination_config: HallucinationConfig,
+) -> None:
+    """
+    Purpose: Verify that context_images produce a multimodal user message.
+    Why this matters: Vision faithfulness needs the rendered pages as image parts.
+    Setup summary: Provide context_images, assert the user message carries text +
+    one image part per data URL.
+    """
+    # Arrange
+    input_data: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        output_text="Output",
+        context_images=[_IMAGE_DATA_URL],
+    )
+
+    # Act
+    result: LanguageModelMessages = _compose_msgs(
+        input_data, hallucination_config, has_context=True
+    )
+
+    # Assert
+    assert len(result.root) == 2
+    user_content = result.root[1].content
+    assert isinstance(user_content, list)
+    text_parts = [p for p in user_content if p.get("type") == "text"]
+    image_parts = [p for p in user_content if p.get("type") == "image_url"]
+    assert len(text_parts) == 1
+    assert len(image_parts) == 1
+    assert image_parts[0]["imageUrl"]["url"] == _IMAGE_DATA_URL
+
+
+@pytest.mark.ai
+def test_compose_msgs__stays_text_only__without_context_images(
+    hallucination_config: HallucinationConfig,
+) -> None:
+    """
+    Purpose: Verify that omitting context_images keeps the exact text-only message.
+    Why this matters: Backward compatibility for all existing (non-vision) callers.
+    Setup summary: No context_images -> user message content is a plain string.
+    """
+    # Arrange
+    input_data: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        context_texts=["Context 1"],
+        output_text="Output",
+    )
+
+    # Act
+    result: LanguageModelMessages = _compose_msgs(
+        input_data, hallucination_config, has_context=True
+    )
+
+    # Assert
+    assert isinstance(result.root[1].content, str)
+
+
+@pytest.mark.ai
+def test_compose_msgs__drops_images__when_model_lacks_vision() -> None:
+    """
+    Purpose: Verify images are dropped when the configured model is not vision-capable.
+    Why this matters: Attaching images to a non-vision model makes the LLM call fail;
+    we must degrade gracefully to a text-only message instead.
+    Setup summary: Configure a text-only model (no VISION capability) with
+    context_images, assert the user message content stays a plain string.
+    """
+    # Arrange
+    text_only_config = HallucinationConfig(
+        enabled=True,
+        language_model=LanguageModelName.AZURE_GPT_4_0613,
+    )
+    assert ModelCapabilities.VISION not in (
+        text_only_config.language_model.capabilities
+    )
+    input_data: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        output_text="Output",
+        context_images=[_IMAGE_DATA_URL],
+    )
+
+    # Act
+    result: LanguageModelMessages = _compose_msgs(
+        input_data, text_only_config, has_context=True
+    )
+
+    # Assert
+    assert isinstance(result.root[1].content, str)
+
+
+@pytest.mark.ai
+def test_get_msgs__treats_context_images_as_context(
+    hallucination_config: HallucinationConfig,
+) -> None:
+    """
+    Purpose: Verify that context_images alone count as context in _get_msgs.
+    Why this matters: A vision-only turn must use the grounded (has_context) prompt
+    and build the multimodal message.
+    Setup summary: Provide only context_images; assert the system prompt matches the
+    has_context variant and the user message is multimodal.
+    """
+    # Arrange
+    image_only: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        output_text="Output",
+        context_images=[_IMAGE_DATA_URL],
+    )
+    grounded: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        context_texts=["Context 1"],
+        output_text="Output",
+    )
+
+    # Act
+    result: LanguageModelMessages = _get_msgs(image_only, hallucination_config)
+    grounded_result: LanguageModelMessages = _get_msgs(grounded, hallucination_config)
+
+    # Assert
+    assert result.root[0].content == grounded_result.root[0].content
+    assert isinstance(result.root[1].content, list)
+
+
+@pytest.mark.ai
+def test_get_msgs__ignores_context_images__when_model_lacks_vision() -> None:
+    """
+    Purpose: Verify image-only input is not treated as context for a non-vision model.
+    Why this matters: _compose_msgs drops the images for a non-vision model, so the
+    grounded prompt would otherwise claim references that are never sent. has_context
+    must reflect the message actually delivered.
+    Setup summary: Configure a text-only model with only context_images; assert the
+    system prompt matches the ungrounded variant and the user message is text-only.
+    """
+    # Arrange
+    text_only_config = HallucinationConfig(
+        enabled=True,
+        language_model=LanguageModelName.AZURE_GPT_4_0613,
+    )
+    image_only: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        output_text="Output",
+        context_images=[_IMAGE_DATA_URL],
+    )
+    no_context: EvaluationMetricInput = EvaluationMetricInput(
+        input_text="Question",
+        output_text="Output",
+    )
+
+    # Act
+    result: LanguageModelMessages = _get_msgs(image_only, text_only_config)
+    no_context_result: LanguageModelMessages = _get_msgs(no_context, text_only_config)
+
+    # Assert
+    assert result.root[0].content == no_context_result.root[0].content
+    assert isinstance(result.root[1].content, str)