unique_toolkit 2026.28.0.dev5__py3-none-any.whl → 2026.28.0.dev7__py3-none-any.whl

unique_toolkit/_common/chunk_relevancy_sorter/service.py

@@ -71,7 +71,7 @@ class ChunkRelevancySorter:
         company_id: str | None = None,
         user_id: str | None = None,
     ):
-        if isinstance(event, (ChatEvent, BaseEvent)):
+        if isinstance(event, BaseEvent):
             self.chunk_relevancy_evaluator = ContextRelevancyEvaluator.from_event(event)
         else:
             [company_id, user_id] = validate_required_values([company_id, user_id])

unique_toolkit/agentic/evaluation/context_relevancy/service.py

@@ -81,7 +81,7 @@ class ContextRelevancyEvaluator:
         company_id: str | None = None,
         user_id: str | None = None,
     ):
-        if isinstance(event, (ChatEvent, BaseEvent)):
+        if isinstance(event, BaseEvent):
             self.language_model_service = LanguageModelService.from_event(event)
         else:
             [company_id, user_id] = validate_required_values([company_id, user_id])

unique_toolkit/agentic/evaluation/hallucination/constants.py

@@ -52,6 +52,18 @@ class HallucinationConfig(EvaluationMetricConfig):
         default={},
         description="Additional options to pass to the language model.",
     )
+    max_external_context_chars_per_source: SkipJsonSchema[int] = Field(
+        default=20_000,
+        ge=0,
+        description=(
+            "Per-source character cap applied to external (non-chunk) "
+            "context texts, e.g. MCP tool output. Raw/large tool results "
+            "inflate the judge prompt and the eval-LLM call; each external "
+            "text is truncated to this many characters before being added "
+            "to the context. 0 disables truncation. Does not affect "
+            "chunk-derived context."
+        ),
+    )
     score_to_label: dict[str, str] = {
         "LOW": "GREEN",
         "MEDIUM": "YELLOW",

unique_toolkit/agentic/evaluation/hallucination/hallucination_evaluation.py

@@ -54,6 +54,17 @@ class HallucinationEvaluation(Evaluation):
             reference_pattern=self.config.reference_pattern,
         )
 
+        # Append non-chunk groundedness context (e.g. MCP tool output). These
+        # are added in full ("all retrieved output as context") rather than via
+        # source-marker selection, since they carry no [sourceN] markers.
+        external_context_texts = self._reference_manager.get_external_context_texts()
+        max_chars = self.config.max_external_context_chars_per_source
+        if max_chars > 0:
+            external_context_texts = [
+                text[:max_chars] for text in external_context_texts
+            ]
+        context_texts = context_texts + external_context_texts
+
         evaluation_result: EvaluationMetricResult = await check_hallucination(
             company_id=self._company_id,
             user_id=self._user_id,

unique_toolkit/agentic/evaluation/tests/test_hallucination_external_context.py

@@ -0,0 +1,122 @@
+"""Tests that HallucinationEvaluation grounds against non-chunk context.
+
+UN-21951: external context texts (e.g. MCP tool output) registered on the
+ReferenceManager must be appended to the evaluation context — in addition to
+chunk-derived context, capped by ``max_external_context_chars_per_source`` —
+without being converted to ContentChunks.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from unique_toolkit.agentic.evaluation.hallucination import (
+    hallucination_evaluation as he,
+)
+from unique_toolkit.agentic.evaluation.hallucination.constants import (
+    HallucinationConfig,
+    SourceSelectionMode,
+)
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+from unique_toolkit.chat.schemas import ChatMessageRole
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+    LanguageModelStreamResponseMessage,
+)
+
+
+def _make_event() -> SimpleNamespace:
+    return SimpleNamespace(
+        company_id="company_1",
+        user_id="user_1",
+        payload=SimpleNamespace(
+            user_message=SimpleNamespace(text="what is ACME revenue?"),
+        ),
+    )
+
+
+def _make_response(text: str = "ACME revenue is 1M.") -> LanguageModelStreamResponse:
+    return LanguageModelStreamResponse(
+        message=LanguageModelStreamResponseMessage(
+            id="msg_1",
+            previous_message_id=None,
+            role=ChatMessageRole.ASSISTANT,
+            text=text,
+            chat_id="chat_1",
+            original_text=text,
+            references=None,
+        ),
+        tool_calls=None,
+    )
+
+
+@pytest.fixture
+def captured_context(monkeypatch: pytest.MonkeyPatch) -> dict:
+    """Patch check_hallucination to capture the context_texts it receives."""
+    captured: dict = {}
+
+    async def _fake_check_hallucination(*, company_id, user_id, input, config):
+        captured["context_texts"] = list(input.context_texts or [])
+        return EvaluationMetricResult(
+            name=EvaluationMetricName.HALLUCINATION,
+            value="LOW",
+            reason="grounded",
+        )
+
+    monkeypatch.setattr(he, "check_hallucination", _fake_check_hallucination)
+    return captured
+
+
+async def test_external_context_is_appended(captured_context: dict) -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["ACME revenue: 1M"])
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(source_selection_mode=SourceSelectionMode.FROM_IDS),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    result = await evaluation.run(_make_response())
+
+    # No chunks → context comes solely from the external (MCP) source.
+    assert captured_context["context_texts"] == ["ACME revenue: 1M"]
+    assert result.value == "LOW"
+    assert result.is_positive is True
+
+
+async def test_external_context_is_truncated(captured_context: dict) -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["x" * 100])
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(
+            source_selection_mode=SourceSelectionMode.FROM_IDS,
+            max_external_context_chars_per_source=10,
+        ),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    await evaluation.run(_make_response())
+
+    assert captured_context["context_texts"] == ["x" * 10]
+
+
+async def test_no_external_context_preserves_chunk_only_behaviour(
+    captured_context: dict,
+) -> None:
+    manager = ReferenceManager()  # no chunks, no external context
+    evaluation = he.HallucinationEvaluation(
+        HallucinationConfig(source_selection_mode=SourceSelectionMode.FROM_IDS),
+        _make_event(),  # type: ignore[arg-type]
+        manager,
+    )
+
+    await evaluation.run(_make_response())
+
+    assert captured_context["context_texts"] == []

unique_toolkit/agentic/reference_manager/reference_manager.py

@@ -32,6 +32,7 @@ class ReferenceManager:
         self._tool_chunks: dict[str, tool_chunks] = {}
         self._chunks: list[ContentChunk] = []
         self._references: list[list[ContentReference]] = []
+        self._external_context_texts: list[str] = []
 
     def extract_referenceable_chunks(
         self, tool_responses: list[ToolCallResponse]
@@ -47,6 +48,21 @@ class ReferenceManager:
     def get_chunks(self) -> list[ContentChunk]:
         return self._chunks
 
+    def add_external_context_texts(self, texts: list[str]) -> None:
+        """Register groundedness context that is not backed by ContentChunks.
+
+        Some retrieval sources (e.g. MCP tool output) provide text the answer
+        is built on but are deliberately *not* modelled as ContentChunks — they
+        are a different entity from ingested document chunks and must not flow
+        through the chunk/reference pipeline. They still need to ground the
+        hallucination check, so they are stored here and appended to the
+        evaluation context alongside the chunk-derived texts.
+        """
+        self._external_context_texts.extend(text for text in texts if text)
+
+    def get_external_context_texts(self) -> list[str]:
+        return self._external_context_texts
+
     def get_tool_chunks(self) -> dict[str, tool_chunks]:
         return self._tool_chunks
 

unique_toolkit/agentic/reference_manager/tests/test_external_context.py

@@ -0,0 +1,35 @@
+"""Tests for non-chunk groundedness context on the ReferenceManager.
+
+External context texts (e.g. MCP tool output, UN-21951) ground the
+hallucination check without being modelled as ContentChunks.
+"""
+
+from unique_toolkit.agentic.reference_manager.reference_manager import (
+    ReferenceManager,
+)
+
+
+def test_external_context_starts_empty() -> None:
+    manager = ReferenceManager()
+    assert manager.get_external_context_texts() == []
+
+
+def test_add_external_context_texts_accumulates() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["a", "b"])
+    manager.add_external_context_texts(["c"])
+    assert manager.get_external_context_texts() == ["a", "b", "c"]
+
+
+def test_add_external_context_texts_drops_empty_strings() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["keep", "", "also"])
+    assert manager.get_external_context_texts() == ["keep", "also"]
+
+
+def test_external_context_is_independent_of_chunks() -> None:
+    manager = ReferenceManager()
+    manager.add_external_context_texts(["mcp output"])
+    # External context must not leak into the chunk pipeline.
+    assert manager.get_chunks() == []
+    assert manager.get_external_context_texts() == ["mcp output"]

unique_toolkit/agentic/tools/experimental/retrieve_search_scope_tool/tool.py

@@ -116,7 +116,7 @@ class RetrieveSearchScopeTool(Tool[RetrieveSearchScopeConfig]):
             )
 
         try:
-            settings = UniqueSettings.from_chat_event(self._event)
+            settings = UniqueSettings.from_chat_event(self.event)
             kb_service = UniqueServiceFactory(
                 settings=settings
             ).knowledge_base_service()

unique_toolkit/agentic/tools/run_context.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import Self
+
+from unique_toolkit.app.schemas import ChatEvent
+
+
+class ToolRunContext(BaseModel):
+    """Tool-manager runtime state decoupled from a raw :class:`ChatEvent`."""
+
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    tool_choices: list[str] = Field(default_factory=list)
+    disabled_tools: list[str] = Field(default_factory=list)
+    tool_init_event: ChatEvent | None = None
+
+    @classmethod
+    def from_chat_event(cls, event: ChatEvent) -> Self:
+        return cls(
+            tool_choices=list(event.payload.tool_choices),
+            disabled_tools=list(event.payload.disabled_tools),
+            tool_init_event=event,
+        )

unique_toolkit/agentic/tools/tool.py

@@ -18,7 +18,8 @@ from unique_toolkit.app.schemas import ChatEvent
 from unique_toolkit.language_model import LanguageModelToolDescription
 
 if TYPE_CHECKING:
-    pass
+    from unique_toolkit.language_model.service import LanguageModelService
+    from unique_toolkit.services.chat_service import ChatService
 from unique_toolkit.language_model.schemas import (
     LanguageModelFunction,
 )
@@ -169,6 +170,9 @@ class Tool(ABC, Generic[ConfigType]):
     def __init__(self, config: ConfigType) -> None: ...
 
     @overload
+    @deprecated(
+        "Passing event is deprecated. Use Tool(config) and inject context in run()."
+    )
     def __init__(
         self,
         config: ConfigType,
@@ -176,17 +180,32 @@ class Tool(ABC, Generic[ConfigType]):
         tool_progress_reporter: ToolProgressReporter | None = ...,
     ) -> None: ...
 
+    @overload
+    def __init__(
+        self,
+        config: ConfigType,
+        *,
+        chat_service: ChatService,
+        language_model_service: LanguageModelService,
+        tool_progress_reporter: ToolProgressReporter | None = ...,
+    ) -> None: ...
+
     def __init__(
         self,
         config: ConfigType,
         event: ChatEvent | None = None,
         tool_progress_reporter: ToolProgressReporter | None = None,
+        *,
+        chat_service: ChatService | None = None,
+        language_model_service: LanguageModelService | None = None,
     ) -> None:
         """Initialize the tool.
 
-        Preferred (decoupled): `Tool(config)` — configuration only.
+        Preferred (decoupled): ``Tool(config)`` — configuration only.
 
-        Backward compatible: `Tool(config, event, tool_progress_reporter)` — creates
+        Service injection: ``Tool(config, chat_service=..., language_model_service=...)``.
+
+        Backward compatible: ``Tool(config, event, tool_progress_reporter)`` — creates
         deprecated chat_service, language_model_service, message_step_logger for
         legacy subclasses.
         """
@@ -200,6 +219,25 @@ class Tool(ABC, Generic[ConfigType]):
         self.logger = getLogger(f"{module_name}.{__name__}")
         self.debug_info: dict[str, Any] = {}
 
+        if (chat_service is None) != (language_model_service is None):
+            raise ValueError(
+                "chat_service and language_model_service must be injected together; "
+                "supplying only one is not supported."
+            )
+
+        if chat_service is not None and language_model_service is not None:
+            from unique_toolkit.agentic.message_log_manager.service import (
+                MessageStepLogger,
+            )
+
+            self._tool_progress_reporter = tool_progress_reporter
+            self._chat_service = chat_service
+            self._language_model_service = language_model_service
+            self._message_step_logger = MessageStepLogger(
+                chat_service=self._chat_service,
+            )
+            return
+
         if event is not None:
             from unique_toolkit.agentic.message_log_manager.service import (
                 MessageStepLogger,
@@ -220,11 +258,12 @@ class Tool(ABC, Generic[ConfigType]):
         "Never reuse event. Dangerous. Prefer Tool(config) and inject context in run()."
     )
     def event(self) -> ChatEvent:
-        if not hasattr(self, "_event"):
+        chat_event = getattr(self, "_event", None)
+        if chat_event is None:
             raise AttributeError(
                 "event not available (tool was initialized with config only). "
             )
-        return self._event
+        return chat_event
 
     @property
     @deprecated(

unique_toolkit/agentic/tools/tool_manager.py

@@ -12,6 +12,7 @@ from openai.types.responses import (
     response_create_params,
 )
 from pydantic import BaseModel, Field
+from typing_extensions import Self
 
 from unique_toolkit._common.execution import (
     Result,
@@ -31,6 +32,7 @@ from unique_toolkit.agentic.tools.openai_builtin.base import (
     OpenAIBuiltInToolName,
 )
 from unique_toolkit.agentic.tools.openai_builtin.manager import OpenAIBuiltInToolManager
+from unique_toolkit.agentic.tools.run_context import ToolRunContext
 from unique_toolkit.agentic.tools.schemas import ToolCallResponse, ToolPrompts
 from unique_toolkit.agentic.tools.tool import Tool
 from unique_toolkit.agentic.tools.tool_progress_reporter import ToolProgressReporter
@@ -88,13 +90,62 @@ class _ToolManager(Generic[_ApiMode]):
         a2a_manager: A2AManager,
         api_mode: _ApiMode,
         builtin_tool_manager: OpenAIBuiltInToolManager | None = None,
+    ) -> None:
+        self._setup(
+            logger=logger,
+            config=config,
+            run_context=ToolRunContext.from_chat_event(event),
+            tool_progress_reporter=tool_progress_reporter,
+            mcp_manager=mcp_manager,
+            a2a_manager=a2a_manager,
+            api_mode=api_mode,
+            builtin_tool_manager=builtin_tool_manager,
+        )
+
+    @classmethod
+    def from_run_context(
+        cls,
+        logger: Logger,
+        config: ToolManagerConfig,
+        *,
+        run_context: ToolRunContext,
+        tool_progress_reporter: ToolProgressReporter,
+        mcp_manager: MCPManager,
+        a2a_manager: A2AManager,
+        api_mode: _ApiMode,
+        builtin_tool_manager: OpenAIBuiltInToolManager | None = None,
+    ) -> Self:
+        instance = cls.__new__(cls)
+        instance._setup(
+            logger=logger,
+            config=config,
+            run_context=run_context,
+            tool_progress_reporter=tool_progress_reporter,
+            mcp_manager=mcp_manager,
+            a2a_manager=a2a_manager,
+            api_mode=api_mode,
+            builtin_tool_manager=builtin_tool_manager,
+        )
+        return instance
+
+    def _setup(
+        self,
+        *,
+        logger: Logger,
+        config: ToolManagerConfig,
+        run_context: ToolRunContext,
+        tool_progress_reporter: ToolProgressReporter,
+        mcp_manager: MCPManager,
+        a2a_manager: A2AManager,
+        api_mode: _ApiMode,
+        builtin_tool_manager: OpenAIBuiltInToolManager | None,
     ) -> None:
         self._logger = logger
         self._config = config
         self._tool_progress_reporter = tool_progress_reporter
         self._tools: list[Tool[Any] | OpenAIBuiltInTool[Any]] = []
-        self._tool_choices = event.payload.tool_choices
-        self._disabled_tools = event.payload.disabled_tools
+        self._tool_choices = run_context.tool_choices
+        self._disabled_tools = run_context.disabled_tools
         self._exclusive_tools = [
             tool.name for tool in self._config.tools if tool.is_exclusive
         ]
@@ -104,17 +155,33 @@ class _ToolManager(Generic[_ApiMode]):
         self._a2a_manager = a2a_manager
         self._builtin_tool_manager = builtin_tool_manager
         self._api_mode = api_mode
-        self._init__tools(event)
-
-    def _init__tools(self, event: ChatEvent) -> None:
+        self._sub_agents: list[SubAgentTool] = []
+        self._builtin_tools: list[OpenAIBuiltInTool[Any]] = []
+        self._mcp_tools: list[Tool[Any]] = []
+        self._internal_tools: list[Tool[Any]] = []
+        self.available_tools: list[
+            Tool[Any] | OpenAIBuiltInTool[Any] | SubAgentTool
+        ] = []
+        self._init__tools(run_context.tool_init_event)
+
+    def _init__tools(self, tool_init_event: ChatEvent | None) -> None:
         tool_choices = self._tool_choices
         tool_configs = self._config.tools
         self._logger.info("Initializing tool definitions...")
         self._logger.info(f"Tool choices: {tool_choices}")
 
-        tool_configs, sub_agents = self._a2a_manager.get_all_sub_agents(
-            tool_configs, event
-        )
+        if tool_init_event is not None:
+            tool_configs, sub_agents = self._a2a_manager.get_all_sub_agents(
+                tool_configs,
+                tool_init_event,
+            )
+        else:
+            tool_configs = [
+                tool_config
+                for tool_config in tool_configs
+                if not tool_config.is_sub_agent
+            ]
+            sub_agents = []
         self._sub_agents = sub_agents
 
         registered_tool_names = set(t.name for t in self._sub_agents)
@@ -133,18 +200,27 @@ class _ToolManager(Generic[_ApiMode]):
         registered_tool_names.update(t.name for t in self._mcp_tools)
 
         # Build internal tools from configurations
-        self._internal_tools = [
-            ToolFactory.build_tool_with_settings(
-                t.name,
-                t,
-                t.configuration,
-                event,
-                tool_progress_reporter=self._tool_progress_reporter,
-            )
-            for t in tool_configs
-            if t.name not in registered_tool_names  # Skip already handled tools
-            and t.name not in OpenAIBuiltInToolName  # Safeguard
-        ]
+        self._internal_tools.clear()
+        for t in tool_configs:
+            if t.name in registered_tool_names:
+                continue
+            if t.name in OpenAIBuiltInToolName:
+                continue
+            if tool_init_event is not None:
+                built_tool = ToolFactory.build_tool_with_settings(
+                    t.name,
+                    t,
+                    t.configuration,
+                    tool_init_event,
+                    tool_progress_reporter=self._tool_progress_reporter,
+                )
+            else:
+                self._logger.info(
+                    "Skipping internal tool '%s' (requires chat event for initialization)",
+                    t.name,
+                )
+                continue
+            self._internal_tools.append(built_tool)
 
         # Combine all types of tools
         self.available_tools = (
@@ -607,6 +683,30 @@ class ToolManager(_ToolManager[Literal["completions"]]):
             builtin_tool_manager=None,
         )
 
+    @classmethod
+    def from_run_context(
+        cls,
+        logger: Logger,
+        config: ToolManagerConfig,
+        *,
+        run_context: ToolRunContext,
+        tool_progress_reporter: ToolProgressReporter,
+        mcp_manager: MCPManager,
+        a2a_manager: A2AManager,
+        api_mode: Literal["completions"] = "completions",
+        builtin_tool_manager: OpenAIBuiltInToolManager | None = None,
+    ) -> Self:
+        return super().from_run_context(
+            logger=logger,
+            config=config,
+            run_context=run_context,
+            tool_progress_reporter=tool_progress_reporter,
+            mcp_manager=mcp_manager,
+            a2a_manager=a2a_manager,
+            api_mode=api_mode,
+            builtin_tool_manager=builtin_tool_manager,
+        )
+
 
 class ResponsesApiToolManager(_ToolManager[Literal["responses"]]):
     def __init__(
@@ -631,6 +731,33 @@ class ResponsesApiToolManager(_ToolManager[Literal["responses"]]):
             builtin_tool_manager=builtin_tool_manager,
         )
 
+    @classmethod
+    def from_run_context(
+        cls,
+        logger: Logger,
+        config: ToolManagerConfig,
+        *,
+        run_context: ToolRunContext,
+        tool_progress_reporter: ToolProgressReporter,
+        mcp_manager: MCPManager,
+        a2a_manager: A2AManager,
+        api_mode: Literal["responses"] = "responses",
+        builtin_tool_manager: OpenAIBuiltInToolManager | None = None,
+    ) -> Self:
+        if builtin_tool_manager is None:
+            msg = "builtin_tool_manager is required for ResponsesApiToolManager"
+            raise ValueError(msg)
+        return super().from_run_context(
+            logger=logger,
+            config=config,
+            run_context=run_context,
+            tool_progress_reporter=tool_progress_reporter,
+            mcp_manager=mcp_manager,
+            a2a_manager=a2a_manager,
+            api_mode=api_mode,
+            builtin_tool_manager=builtin_tool_manager,
+        )
+
     def get_required_include_params(self) -> list[ResponseIncludable]:
         """Return Responses API include params required by all active built-in tools."""
         return self._builtin_tool_manager.get_required_include_params()