unique_toolkit 2026.28.0.dev5__py3-none-any.whl → 2026.28.0.dev6__py3-none-any.whl

unique_toolkit/agentic/evaluation/hallucination/constants.py

@@ -52,6 +52,18 @@ class HallucinationConfig(EvaluationMetricConfig):
         default={},
         description="Additional options to pass to the language model.",
     )
+    max_external_context_chars_per_source: SkipJsonSchema[int] = Field(
+        default=20_000,
+        ge=0,
+        description=(
+            "Per-source character cap applied to external (non-chunk) "
+            "context texts, e.g. MCP tool output. Raw/large tool results "
+            "inflate the judge prompt and the eval-LLM call; each external "
+            "text is truncated to this many characters before being added "
+            "to the context. 0 disables truncation. Does not affect "
+            "chunk-derived context."
+        ),
+    )
     score_to_label: dict[str, str] = {
         "LOW": "GREEN",
         "MEDIUM": "YELLOW",

unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py

@@ -54,6 +54,17 @@ class HallucinationEvaluation(Evaluation):
             reference_pattern=self.config.reference_pattern,
         )
 
+        # Append non-chunk groundedness context (e.g. MCP tool output). These
+        # are added in full ("all retrieved output as context") rather than via
+        # source-marker selection, since they carry no [sourceN] markers.
+        external_context_texts = self._reference_manager.get_external_context_texts()
+        max_chars = self.config.max_external_context_chars_per_source
+        if max_chars > 0:
+            external_context_texts = [
+                text[:max_chars] for text in external_context_texts
+            ]
+        context_texts = context_texts + external_context_texts
+
         evaluation_result: EvaluationMetricResult = await check_hallucination(
             company_id=self._company_id,
             user_id=self._user_id,

unique_toolkit/agentic/evaluation/tests/test_hallucination_external_context.py

@@ -0,0 +1,122 @@
+"""Tests that HallucinationEvaluation grounds against non-chunk context.
+
+UN-21951: external context texts (e.g. MCP tool output) registered on the
+ReferenceManager must be appended to the evaluation context — in addition to
+chunk-derived context, capped by ``max_external_context_chars_per_source`` —
+without being converted to ContentChunks.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from unique_toolkit.agentic.evaluation.hallucination import (
+    hallucination_evaluation as he,
+)
+from unique_toolkit.agentic.evaluation.hallucination.constants import (
+    HallucinationConfig,
+    SourceSelectionMode,
+)
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+from unique_toolkit.chat.schemas import ChatMessageRole
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+    LanguageModelStreamResponseMessage,
+)
+
+
+def _make_event() -> SimpleNamespace:
+    return SimpleNamespace(
+        company_id="company_1",
+        user_id="user_1",
+        payload=SimpleNamespace(
+            user_message=SimpleNamespace(text="what is ACME revenue?"),
+        ),
+    )
+
+
+def _make_response(text: str = "ACME revenue is 1M.") -> LanguageModelStreamResponse:
+    return LanguageModelStreamResponse(
+        message=LanguageModelStreamResponseMessage(
+            id="msg_1",
+            previous_message_id=None,
+            role=ChatMessageRole.ASSISTANT,
+            text=text,
+            chat_id="chat_1",
+            original_text=text,
+            references=None,
+        ),
+        tool_calls=None,
+    )
+
+
+@pytest.fixture
+def captured_context(monkeypatch: pytest.MonkeyPatch) -> dict:
+    """Patch check_hallucination to capture the context_texts it receives."""
+    captured: dict = {}
+
+    async def _fake_check_hallucination(*, company_id, user_id, input, config):
+        captured["context_texts"] = list(input.context_texts or [])
+        return EvaluationMetricResult(
+            name=EvaluationMetricName.HALLUCINATION,
+            value="LOW",
+            reason="grounded",
+        )
+
+    monkeypatch.setattr(he, "check_hallucination", _fake_check_hallucination)
+    return captured
+
+
+async def test_external_context_is_appended(captured_context: dict) -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["ACME revenue: 1M"])
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(source_selection_mode=SourceSelectionMode.FROM_IDS),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    result = await evaluation.run(_make_response())
+
+    # No chunks → context comes solely from the external (MCP) source.
+    assert captured_context["context_texts"] == ["ACME revenue: 1M"]
+    assert result.value == "LOW"
+    assert result.is_positive is True
+
+
+async def test_external_context_is_truncated(captured_context: dict) -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["x" * 100])
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(
+            source_selection_mode=SourceSelectionMode.FROM_IDS,
+            max_external_context_chars_per_source=10,
+        ),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    await evaluation.run(_make_response())
+
+    assert captured_context["context_texts"] == ["x" * 10]
+
+
+async def test_no_external_context_preserves_chunk_only_behaviour(
+    captured_context: dict,
+) -> None:
+    manager = ReferenceManager()  # no chunks, no external context
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(source_selection_mode=SourceSelectionMode.FROM_IDS),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    await evaluation.run(_make_response())
+
+    assert captured_context["context_texts"] == []

unique_toolkit/agentic/reference_manager/reference_manager.py

@@ -32,6 +32,7 @@ class ReferenceManager:
         self._tool_chunks: dict[str, tool_chunks] = {}
         self._chunks: list[ContentChunk] = []
         self._references: list[list[ContentReference]] = []
+        self._external_context_texts: list[str] = []
 
     def extract_referenceable_chunks(
         self, tool_responses: list[ToolCallResponse]
@@ -47,6 +48,21 @@ class ReferenceManager:
     def get_chunks(self) -> list[ContentChunk]:
         return self._chunks
 
+    def add_external_context_texts(self, texts: list[str]) -> None:
+        """Register groundedness context that is not backed by ContentChunks.
+
+        Some retrieval sources (e.g. MCP tool output) provide text the answer
+        is built on but are deliberately *not* modelled as ContentChunks — they
+        are a different entity from ingested document chunks and must not flow
+        through the chunk/reference pipeline. They still need to ground the
+        hallucination check, so they are stored here and appended to the
+        evaluation context alongside the chunk-derived texts.
+        """
+        self._external_context_texts.extend(text for text in texts if text)
+
+    def get_external_context_texts(self) -> list[str]:
+        return self._external_context_texts
+
     def get_tool_chunks(self) -> dict[str, tool_chunks]:
         return self._tool_chunks
 

unique_toolkit/agentic/reference_manager/tests/test_external_context.py

@@ -0,0 +1,35 @@
+"""Tests for non-chunk groundedness context on the ReferenceManager.
+
+External context texts (e.g. MCP tool output, UN-21951) ground the
+hallucination check without being modelled as ContentChunks.
+"""
+
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+
+
+def test_external_context_starts_empty() -> None:
+    manager = ReferenceManager()
+    assert manager.get_external_context_texts() == []
+
+
+def test_add_external_context_texts_accumulates() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["a", "b"])
+    manager.add_external_context_texts(["c"])
+    assert manager.get_external_context_texts() == ["a", "b", "c"]
+
+
+def test_add_external_context_texts_drops_empty_strings() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["keep", "", "also"])
+    assert manager.get_external_context_texts() == ["keep", "also"]
+
+
+def test_external_context_is_independent_of_chunks() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["mcp output"])
+    # External context must not leak into the chunk pipeline.
+    assert manager.get_chunks() == []
+    assert manager.get_external_context_texts() == ["mcp output"]

unique_toolkit/content/schemas.py

@@ -215,7 +215,15 @@ class ContentReference(BaseModel):
     sequence_number: int
     source: str
     source_id: str
-    url: str
+    url: str | None = Field(
+        default=None,
+        description=(
+            "Link target for the reference. Optional: some sources (e.g. an "
+            "opaque MCP tool result with no resource link) have no URL and "
+            "render as a display-only, non-clickable citation. Nullable "
+            "end-to-end (node-chat persists it as a nullable column)."
+        ),
+    )
     original_index: list[int] = Field(
         default=[],
         description="List of indices in the ChatMessage original_text this reference refers to. This is usually the id in the functionCallResponse. List type due to implementation in node-chat",

{unique_toolkit-2026.28.0.dev5.dist-info → unique_toolkit-2026.28.0.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 2026.28.0.dev5
+Version: 2026.28.0.dev6
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
 Requires-Dist: openai>=1.105.0,<3
 Requires-Dist: platformdirs>=4.0.0,<5
 Requires-Dist: pillow>=12.1.1
-Requires-Dist: unique-sdk>=2026.28.0.dev6,<2026.28.0rc0
+Requires-Dist: unique-sdk>=2026.28.0.dev8,<2026.28.0rc0
 Requires-Dist: jambo>=0.1.2,<0.2
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: docxtpl>=0.20.1,<0.21

{unique_toolkit-2026.28.0.dev5.dist-info → unique_toolkit-2026.28.0.dev6.dist-info}/RECORD

@@ -93,8 +93,8 @@ unique_toolkit/agentic/evaluation/context_relevancy/schema.py,sha256=lZd0TPzH43i
 unique_toolkit/agentic/evaluation/context_relevancy/service.py,sha256=i6WA_BwHyhHDfnvK59UC0z1FgZ8EPW5OJyYLbSxNRic,8760
 unique_toolkit/agentic/evaluation/evaluation_manager.py,sha256=4HeKsDg6tLkJXNw3kzjmVLbD-86IRzKLQSuPwYtfYBk,7992
 unique_toolkit/agentic/evaluation/exception.py,sha256=7lcVbCyoN4Md1chNJDFxpUYyWbVrcr9dcc3TxWykJTc,115
-unique_toolkit/agentic/evaluation/hallucination/constants.py,sha256=CBWc2CEvSrTyzn1KryxDp7-vp9RLgF16WJ7Ch2JcWNY,2652
-unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py,sha256=Fj4FxKtiY91V5P_IJGlUdmJBT7GJ93SelR6XGoQJUQU,3949
+unique_toolkit/agentic/evaluation/hallucination/constants.py,sha256=uEQR-CNKmeXizDaLOLHhwTZ0iYUmFwoKeMLHPm4M0sY,3208
+unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py,sha256=HvaaRJ4D4pz_77IT2ZbL-OMfVSEOcmcalecOPnBsiRI,4548
 unique_toolkit/agentic/evaluation/hallucination/prompts/__init__.py,sha256=4KFYMZsB3fJUKzoiUJE1npZ0gueWgvceB32EUrN-v7A,343
 unique_toolkit/agentic/evaluation/hallucination/prompts/system_prompt.j2,sha256=sDUX6G645Ba40D_qKu4cUI8g-sJOfG8JpZreTNFgf7M,2616
 unique_toolkit/agentic/evaluation/hallucination/prompts/user_prompt.j2,sha256=mD_qE9fOkyc1XXrebFt097ddx8bTlA6lbY04hKSQmWs,273
@@ -106,6 +106,7 @@ unique_toolkit/agentic/evaluation/tests/fixtures.py,sha256=Q-ughTfDiAdsMKbBVGzFi
 unique_toolkit/agentic/evaluation/tests/test_config.py,sha256=p7xFQ7KE_yU8jGpqYA7ntAYe5Vln33wd6nwv3FM9XfI,8327
 unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py,sha256=NcSOyBJ_lqYehtlraZPo9RLutCitTP76kvkuyogSD2A,9477
 unique_toolkit/agentic/evaluation/tests/test_hallucination_constants.py,sha256=A77nRcniI7hUf640f1sTvCAibQbJE30ikbz1RX4fltw,20478
+unique_toolkit/agentic/evaluation/tests/test_hallucination_external_context.py,sha256=mIHxQDxW7oUbvOGH5bMcoV_yKr9BZvJlgE0JYicLdsM,3949
 unique_toolkit/agentic/evaluation/tests/test_hallucination_utils.py,sha256=ogds7jUwwGJHdabhayhEY_1JbiaVv7Fxh58TMwgbqgw,37537
 unique_toolkit/agentic/evaluation/tests/test_output_parser.py,sha256=KfltytmvqnPWLhmZpBXqcRmnlYorw_USwM5rkLVv8so,5179
 unique_toolkit/agentic/evaluation/tests/test_prompt_loaders.py,sha256=zBREdlKf5tdDyB8XSaNgpQv3-tuZJoYteeJrp6WMWDM,11897
@@ -138,7 +139,8 @@ unique_toolkit/agentic/message_log_manager/__init__.py,sha256=3-KY_sGkPbNoSnrzwP
 unique_toolkit/agentic/message_log_manager/service.py,sha256=yXeV_70ATUJrFi_KYRJ8PRmRbo27kQjY5cjazxDFkpI,9840
 unique_toolkit/agentic/message_log_order.py,sha256=GaZOlAkFS2BpTc72af6Nit9inm6LskBYTze5Q7dd9cs,1041
 unique_toolkit/agentic/postprocessor/postprocessor_manager.py,sha256=zOKnyzt1JwN7Godb4g3RQUOvwaR-Eg4ygm66gkmDM50,8125
-unique_toolkit/agentic/reference_manager/reference_manager.py,sha256=x51CT0D8HHu2LzgXdHGy0leOYpjnsxVbPZ2nc28G9mA,4005
+unique_toolkit/agentic/reference_manager/reference_manager.py,sha256=axqjSNFoR6oGVKg_fK1D2IsA9sxvreyPutRxK17dBxI,4844
+unique_toolkit/agentic/reference_manager/tests/test_external_context.py,sha256=FlCr8rcvrJDkVbKW6pDylSTmP0v0luH_3TGBvXm4KZw,1246
 unique_toolkit/agentic/responses_api/__init__.py,sha256=9WTO-ef7fGE9Y1QtZJFm8Q_jkwK8Srtl-HWvpAD2Wxs,668
 unique_toolkit/agentic/responses_api/postprocessors/code_display.py,sha256=CboLgZ4Rncz_kxRwqoq2C93pZsdylHK4jU145YodEkM,936
 unique_toolkit/agentic/responses_api/postprocessors/generated_files.py,sha256=yVwFT8ObMVJZpUufO1jEHofAdrXm5ndsqbSepuUkNPE,1064
@@ -254,7 +256,7 @@ unique_toolkit/chat/utils.py,sha256=CsD9IER2gA1IOV0fmUOl-N0LmKvA7JP9dGrOqeOY0Gg,
 unique_toolkit/content/__init__.py,sha256=nnQDfnNTEvI8-3dgWvIojoFDYU7edwd8Kloi6lMDG0M,1586
 unique_toolkit/content/constants.py,sha256=1iy4Y67xobl5VTnJB6SxSyuoBWbdLl9244xfVMUZi5o,60
 unique_toolkit/content/functions.py,sha256=avnlEQ1FkVz2NTbXUkafU04jzygqOJtUMQL605rEL4c,34019
-unique_toolkit/content/schemas.py,sha256=iBorQu7OM9d1gNwky1oJv7krsAqbTst2_TrE7cQH0GA,10290
+unique_toolkit/content/schemas.py,sha256=00qbe-DlUjqlf-n3QVOotZjpsj0LZNnUoPmdwyMjVHY,10667
 unique_toolkit/content/service.py,sha256=fR93_76aPczx2voxR7cvYDs_-pxD-Mc9WP6-5IqA8ko,28301
 unique_toolkit/content/smart_rules.py,sha256=9GlHXm63POTi4jy5hW0qUjeqy-JSeL1wIbURM1GKhc4,11481
 unique_toolkit/content/utils.py,sha256=keDkqu_ZzQY0y_g7g_XLOza3p4XsvfqVVDI9ylM91KY,10115
@@ -402,6 +404,6 @@ unique_toolkit/short_term_memory/service.py,sha256=Qo-c9yxzC1ZEEHJyIG_vnRs_ef0SK
 unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/smart_rules/compile.py,sha256=vRe0aMJWuVIK-VsqomgxQImzke7MDmwgSxxJfYCa7zw,1386
 unique_toolkit/test_utilities/events.py,sha256=FLrhMh0A_mQutAUjoD0FdMcVQ4Gq31r35OVGzRjvLro,6760
-unique_toolkit-2026.28.0.dev5.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
-unique_toolkit-2026.28.0.dev5.dist-info/METADATA,sha256=2C5KgFuTF2BY0kFsCSw0GLePoG1n6mLY7eIxAbsKl04,7426
-unique_toolkit-2026.28.0.dev5.dist-info/RECORD,,
+unique_toolkit-2026.28.0.dev6.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
+unique_toolkit-2026.28.0.dev6.dist-info/METADATA,sha256=hoO0ZUgIV4theG_01eSZ0nk_8iU5vBqbxVk5St6xIW0,7426
+unique_toolkit-2026.28.0.dev6.dist-info/RECORD,,