unique-toolkit 2026.28.0.dev4__tar.gz → 2026.28.0.dev6__tar.gz

{unique_toolkit-2026.28.0.dev4 → unique_toolkit-2026.28.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 2026.28.0.dev4
+Version: 2026.28.0.dev6
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
 Requires-Dist: openai>=1.105.0,<3
 Requires-Dist: platformdirs>=4.0.0,<5
 Requires-Dist: pillow>=12.1.1
-Requires-Dist: unique-sdk>=2026.28.0.dev5,<2026.28.0rc0
+Requires-Dist: unique-sdk>=2026.28.0.dev8,<2026.28.0rc0
 Requires-Dist: jambo>=0.1.2,<0.2
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: docxtpl>=0.20.1,<0.21

{unique_toolkit-2026.28.0.dev4 → unique_toolkit-2026.28.0.dev6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "2026.28.0.dev4"
+version = "2026.28.0.dev6"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,7 +28,7 @@ dependencies = [
     "openai>=1.105.0,<3",
     "platformdirs>=4.0.0,<5",
     "pillow>=12.1.1",
-    "unique-sdk>=2026.28.0.dev5,<2026.28.0rc0",
+    "unique-sdk>=2026.28.0.dev8,<2026.28.0rc0",
     "jambo>=0.1.2,<0.2",
     "jinja2>=3.1.6,<4",
     "docxtpl>=0.20.1,<0.21",

{unique_toolkit-2026.28.0.dev4 → unique_toolkit-2026.28.0.dev6}/unique_toolkit/agentic/evaluation/hallucination/constants.py

@@ -52,6 +52,18 @@ class HallucinationConfig(EvaluationMetricConfig):
         default={},
         description="Additional options to pass to the language model.",
     )
+    max_external_context_chars_per_source: SkipJsonSchema[int] = Field(
+        default=20_000,
+        ge=0,
+        description=(
+            "Per-source character cap applied to external (non-chunk) "
+            "context texts, e.g. MCP tool output. Raw/large tool results "
+            "inflate the judge prompt and the eval-LLM call; each external "
+            "text is truncated to this many characters before being added "
+            "to the context. 0 disables truncation. Does not affect "
+            "chunk-derived context."
+        ),
+    )
     score_to_label: dict[str, str] = {
         "LOW": "GREEN",
         "MEDIUM": "YELLOW",

{unique_toolkit-2026.28.0.dev4 → unique_toolkit-2026.28.0.dev6}/unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py

@@ -54,6 +54,17 @@ class HallucinationEvaluation(Evaluation):
             reference_pattern=self.config.reference_pattern,
         )
 
+        # Append non-chunk groundedness context (e.g. MCP tool output). These
+        # are added in full ("all retrieved output as context") rather than via
+        # source-marker selection, since they carry no [sourceN] markers.
+        external_context_texts = self._reference_manager.get_external_context_texts()
+        max_chars = self.config.max_external_context_chars_per_source
+        if max_chars > 0:
+            external_context_texts = [
+                text[:max_chars] for text in external_context_texts
+            ]
+        context_texts = context_texts + external_context_texts
+
         evaluation_result: EvaluationMetricResult = await check_hallucination(
             company_id=self._company_id,
             user_id=self._user_id,

unique_toolkit-2026.28.0.dev6/unique_toolkit/agentic/evaluation/tests/test_hallucination_external_context.py

@@ -0,0 +1,122 @@
+"""Tests that HallucinationEvaluation grounds against non-chunk context.
+
+UN-21951: external context texts (e.g. MCP tool output) registered on the
+ReferenceManager must be appended to the evaluation context — in addition to
+chunk-derived context, capped by ``max_external_context_chars_per_source`` —
+without being converted to ContentChunks.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from unique_toolkit.agentic.evaluation.hallucination import (
+    hallucination_evaluation as he,
+)
+from unique_toolkit.agentic.evaluation.hallucination.constants import (
+    HallucinationConfig,
+    SourceSelectionMode,
+)
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+from unique_toolkit.chat.schemas import ChatMessageRole
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+    LanguageModelStreamResponseMessage,
+)
+
+
+def _make_event() -> SimpleNamespace:
+    return SimpleNamespace(
+        company_id="company_1",
+        user_id="user_1",
+        payload=SimpleNamespace(
+            user_message=SimpleNamespace(text="what is ACME revenue?"),
+        ),
+    )
+
+
+def _make_response(text: str = "ACME revenue is 1M.") -> LanguageModelStreamResponse:
+    return LanguageModelStreamResponse(
+        message=LanguageModelStreamResponseMessage(
+            id="msg_1",
+            previous_message_id=None,
+            role=ChatMessageRole.ASSISTANT,
+            text=text,
+            chat_id="chat_1",
+            original_text=text,
+            references=None,
+        ),
+        tool_calls=None,
+    )
+
+
+@pytest.fixture
+def captured_context(monkeypatch: pytest.MonkeyPatch) -> dict:
+    """Patch check_hallucination to capture the context_texts it receives."""
+    captured: dict = {}
+
+    async def _fake_check_hallucination(*, company_id, user_id, input, config):
+        captured["context_texts"] = list(input.context_texts or [])
+        return EvaluationMetricResult(
+            name=EvaluationMetricName.HALLUCINATION,
+            value="LOW",
+            reason="grounded",
+        )
+
+    monkeypatch.setattr(he, "check_hallucination", _fake_check_hallucination)
+    return captured
+
+
+async def test_external_context_is_appended(captured_context: dict) -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["ACME revenue: 1M"])
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(source_selection_mode=SourceSelectionMode.FROM_IDS),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    result = await evaluation.run(_make_response())
+
+    # No chunks → context comes solely from the external (MCP) source.
+    assert captured_context["context_texts"] == ["ACME revenue: 1M"]
+    assert result.value == "LOW"
+    assert result.is_positive is True
+
+
+async def test_external_context_is_truncated(captured_context: dict) -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["x" * 100])
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(
+            source_selection_mode=SourceSelectionMode.FROM_IDS,
+            max_external_context_chars_per_source=10,
+        ),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    await evaluation.run(_make_response())
+
+    assert captured_context["context_texts"] == ["x" * 10]
+
+
+async def test_no_external_context_preserves_chunk_only_behaviour(
+    captured_context: dict,
+) -> None:
+    manager = ReferenceManager()  # no chunks, no external context
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(source_selection_mode=SourceSelectionMode.FROM_IDS),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    await evaluation.run(_make_response())
+
+    assert captured_context["context_texts"] == []

{unique_toolkit-2026.28.0.dev4 → unique_toolkit-2026.28.0.dev6}/unique_toolkit/agentic/reference_manager/reference_manager.py

@@ -32,6 +32,7 @@ class ReferenceManager:
         self._tool_chunks: dict[str, tool_chunks] = {}
         self._chunks: list[ContentChunk] = []
         self._references: list[list[ContentReference]] = []
+        self._external_context_texts: list[str] = []
 
     def extract_referenceable_chunks(
         self, tool_responses: list[ToolCallResponse]
@@ -47,6 +48,21 @@ class ReferenceManager:
     def get_chunks(self) -> list[ContentChunk]:
         return self._chunks
 
+    def add_external_context_texts(self, texts: list[str]) -> None:
+        """Register groundedness context that is not backed by ContentChunks.
+
+        Some retrieval sources (e.g. MCP tool output) provide text the answer
+        is built on but are deliberately *not* modelled as ContentChunks — they
+        are a different entity from ingested document chunks and must not flow
+        through the chunk/reference pipeline. They still need to ground the
+        hallucination check, so they are stored here and appended to the
+        evaluation context alongside the chunk-derived texts.
+        """
+        self._external_context_texts.extend(text for text in texts if text)
+
+    def get_external_context_texts(self) -> list[str]:
+        return self._external_context_texts
+
     def get_tool_chunks(self) -> dict[str, tool_chunks]:
         return self._tool_chunks
 

unique_toolkit-2026.28.0.dev6/unique_toolkit/agentic/reference_manager/tests/test_external_context.py

@@ -0,0 +1,35 @@
+"""Tests for non-chunk groundedness context on the ReferenceManager.
+
+External context texts (e.g. MCP tool output, UN-21951) ground the
+hallucination check without being modelled as ContentChunks.
+"""
+
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+
+
+def test_external_context_starts_empty() -> None:
+    manager = ReferenceManager()
+    assert manager.get_external_context_texts() == []
+
+
+def test_add_external_context_texts_accumulates() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["a", "b"])
+    manager.add_external_context_texts(["c"])
+    assert manager.get_external_context_texts() == ["a", "b", "c"]
+
+
+def test_add_external_context_texts_drops_empty_strings() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["keep", "", "also"])
+    assert manager.get_external_context_texts() == ["keep", "also"]
+
+
+def test_external_context_is_independent_of_chunks() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["mcp output"])
+    # External context must not leak into the chunk pipeline.
+    assert manager.get_chunks() == []
+    assert manager.get_external_context_texts() == ["mcp output"]

{unique_toolkit-2026.28.0.dev4 → unique_toolkit-2026.28.0.dev6}/unique_toolkit/content/schemas.py

@@ -215,7 +215,15 @@ class ContentReference(BaseModel):
     sequence_number: int
     source: str
     source_id: str
-    url: str
+    url: str | None = Field(
+        default=None,
+        description=(
+            "Link target for the reference. Optional: some sources (e.g. an "
+            "opaque MCP tool result with no resource link) have no URL and "
+            "render as a display-only, non-clickable citation. Nullable "
+            "end-to-end (node-chat persists it as a nullable column)."
+        ),
+    )
     original_index: list[int] = Field(
         default=[],
         description="List of indices in the ChatMessage original_text this reference refers to. This is usually the id in the functionCallResponse. List type due to implementation in node-chat",