unique_toolkit 1.3.2__tar.gz → 1.4.0__tar.gz

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0] - 2025-09-29
+- Add ability to consolidate sub agent's assessments.
+
+## [1.3.3] - 2025-09-30
+- fix bug in exclusive tools not making them selectable
+
 ## [1.3.2] - 2025-09-29
 - Auto-update unique settings on event arrival in SSE client
 

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.3.2
+Version: 1.4.0
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -118,6 +118,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0] - 2025-09-29
+- Add ability to consolidate sub agent's assessments.
+
+## [1.3.3] - 2025-09-30
+- fix bug in exclusive tools not making them selectable
+
 ## [1.3.2] - 2025-09-29
 - Auto-update unique settings on event arrival in SSE client
 

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "1.3.2"
+version = "1.4.0"
 description = ""
 authors = [
     "Cedric Klinkert <cedric.klinkert@unique.ch>",

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/unique_toolkit/agentic/evaluation/schemas.py

@@ -15,6 +15,7 @@ from unique_toolkit.chat.schemas import (
 class EvaluationMetricName(StrEnum):
     HALLUCINATION = "hallucination"
     CONTEXT_RELEVANCY = "relevancy"
+    SUB_AGENT = "sub_agent"
 
 
 class EvaluationMetricInputFieldName(StrEnum):

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/unique_toolkit/agentic/tools/a2a/config.py

@@ -23,6 +23,11 @@ class SubAgentToolDisplayConfig(BaseModel):
     remove_from_history: bool = True
 
 
+class SubAgentEvaluationConfig(BaseModel):
+    model_config = get_configuration_dict()
+    display_evalution: bool = True
+
+
 class SubAgentToolConfig(BaseToolConfig):
     model_config = get_configuration_dict()
 
@@ -43,3 +48,4 @@ class SubAgentToolConfig(BaseToolConfig):
     max_wait: float = 120.0
 
     response_display_config: SubAgentToolDisplayConfig = SubAgentToolDisplayConfig()
+    evaluation_config: SubAgentEvaluationConfig = SubAgentEvaluationConfig()

unique_toolkit-1.4.0/unique_toolkit/agentic/tools/a2a/evaluation/__init__.py

@@ -0,0 +1,6 @@
+from unique_toolkit.agentic.tools.a2a.evaluation.config import SubAgentEvaluationConfig
+from unique_toolkit.agentic.tools.a2a.evaluation.evaluator import (
+    SubAgentsEvaluation,
+)
+
+__all__ = ["SubAgentsEvaluation", "SubAgentEvaluationConfig"]

unique_toolkit-1.4.0/unique_toolkit/agentic/tools/a2a/evaluation/config.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from unique_toolkit._common.default_language_model import DEFAULT_GPT_4o
+from unique_toolkit._common.validators import LMI, get_LMI_default_field
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentType,
+)
+
+DEFAULT_EVALUATION_SYSTEM_MESSAGE_TEMPLATE = """
+You are a through and precise summarization model.
+You will receive a list of "assessments" of one or more agent(s) response(s).
+Your task is to give a brief summary (1-10 sentences) of the received assessments, following the following guidelines:
+1. You must NOT in ANY case state a fact that is not stated in the given assessments.
+2. You must focus first and foremost on the failing assessments, labeled `RED` below.
+3. You must mention each agent's name when summarizing its list of assessments.
+""".strip()
+
+with open(Path(__file__).parent / "summarization_user_message.j2", "r") as file:
+    DEFAULT_SUMMARIZATION_USER_MESSAGE_TEMPLATE = file.read().strip()
+
+
+class SubAgentEvaluationConfig(BaseModel):
+    assessment_type: ChatMessageAssessmentType = Field(
+        default=ChatMessageAssessmentType.COMPLIANCE,
+        description="The type of assessment to use in the display.",
+    )
+
+    summarization_model: LMI = get_LMI_default_field(DEFAULT_GPT_4o)
+    summarization_system_message: str = Field(
+        default=DEFAULT_EVALUATION_SYSTEM_MESSAGE_TEMPLATE,
+        description="The system message template for the summarization model.",
+    )
+    summarization_user_message_template: str = Field(
+        default=DEFAULT_SUMMARIZATION_USER_MESSAGE_TEMPLATE,
+        description="The user message template for the summarization model.",
+    )

unique_toolkit-1.4.0/unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py

@@ -0,0 +1,175 @@
+import logging
+from collections import defaultdict
+from typing import override
+
+import unique_sdk
+from jinja2 import Template
+from typing_extensions import TypedDict
+
+from unique_toolkit.agentic.evaluation.evaluation_manager import Evaluation
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.tools.a2a.evaluation.config import SubAgentEvaluationConfig
+from unique_toolkit.agentic.tools.a2a.service import SubAgentTool
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.language_model.builder import MessagesBuilder
+from unique_toolkit.language_model.schemas import LanguageModelStreamResponse
+from unique_toolkit.language_model.service import LanguageModelService
+
+logger = logging.getLogger(__name__)
+
+
+class _SubAgentToolInfo(TypedDict):
+    assessment: list[unique_sdk.Space.Assessment] | None
+    display_name: str
+
+
+class SubAgentsEvaluation(Evaluation):
+    DISPLAY_NAME = "Sub Agents"
+
+    def __init__(
+        self,
+        config: SubAgentEvaluationConfig,
+        sub_agent_tools: list[SubAgentTool],
+        language_model_service: LanguageModelService,
+    ):
+        super().__init__(EvaluationMetricName.SUB_AGENT)
+        self._config = config
+
+        self._assistant_id_to_tool_info: dict[str, _SubAgentToolInfo] = {}
+        self._language_model_service = language_model_service
+
+        for sub_agent_tool in sub_agent_tools:
+            if sub_agent_tool.config.evaluation_config.display_evalution:
+                sub_agent_tool.subscribe(self)
+                self._assistant_id_to_tool_info[sub_agent_tool.config.assistant_id] = {
+                    "assessment": None,
+                    "display_name": sub_agent_tool.display_name(),
+                }
+
+    @override
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        return self._config.assessment_type
+
+    @override
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:
+        logger.info("Running sub agents evaluation")
+
+        sub_agents_display_data = []
+
+        value = ChatMessageAssessmentLabel.GREEN
+
+        label_comparison_dict = defaultdict(
+            lambda: 3
+        )  # Unkown labels are highest in the sorting
+        label_comparison_dict[ChatMessageAssessmentLabel.GREEN] = 2
+        label_comparison_dict[ChatMessageAssessmentLabel.YELLOW] = 1
+        label_comparison_dict[ChatMessageAssessmentLabel.RED] = 0
+
+        for assistant_id, tool_info in self._assistant_id_to_tool_info.items():
+            assessments = tool_info["assessment"]
+            if assessments is None or len(assessments) == 0:
+                logger.info("No assessment found for assistant %s", assistant_id)
+                continue
+
+            assessments_display_data = sorted(
+                assessments, key=lambda x: label_comparison_dict[x["label"]]
+            )
+
+            for assessment in assessments:
+                if label := assessment["label"]:
+                    if label not in ChatMessageAssessmentLabel:
+                        logger.warning(
+                            "Unkown assistant label %s for assistant %s will be ignored",
+                            label,
+                            assistant_id,
+                        )
+                        continue
+                    value = min(value, label, key=lambda x: label_comparison_dict[x])
+
+            sub_agents_display_data.append(
+                {
+                    "name": tool_info["display_name"],
+                    "assessments": assessments_display_data,
+                }
+            )
+
+        should_summarize = False
+        reason = ""
+
+        if len(sub_agents_display_data) > 1:
+            should_summarize = True
+        elif len(sub_agents_display_data) == 1:
+            if len(sub_agents_display_data[0]["assessments"]) > 1:
+                should_summarize = True
+            else:
+                reason = (
+                    sub_agents_display_data[0]["assessments"][0]["explanation"] or ""
+                )
+        else:
+            assert False, "No sub agents assessments found"
+
+        if should_summarize:
+            messages = (
+                MessagesBuilder()
+                .system_message_append(self._config.summarization_system_message)
+                .user_message_append(
+                    Template(self._config.summarization_user_message_template).render(
+                        sub_agents=sub_agents_display_data,
+                    )
+                )
+                .build()
+            )
+
+            reason = await self._language_model_service.complete_async(
+                messages=messages,
+                model_name=self._config.summarization_model.name,
+                temperature=0.0,
+            )
+            reason = str(reason.choices[0].message.content)
+
+        return EvaluationMetricResult(
+            name=self.get_name(),
+            value=value,
+            reason=reason,
+            is_positive=value == ChatMessageAssessmentLabel.GREEN,
+        )
+
+    @override
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        return EvaluationAssessmentMessage(
+            status=ChatMessageAssessmentStatus.DONE,
+            explanation=evaluation_result.reason,
+            title=self.DISPLAY_NAME,
+            label=evaluation_result.value,  # type: ignore
+            type=self.get_assessment_type(),
+        )
+
+    def notify_sub_agent_response(
+        self, sub_agent_assistant_id: str, response: unique_sdk.Space.Message
+    ) -> None:
+        if sub_agent_assistant_id not in self._assistant_id_to_tool_info:
+            logger.warning(
+                "Unknown assistant id %s received, assessment will be ignored.",
+                sub_agent_assistant_id,
+            )
+            return
+
+        self._assistant_id_to_tool_info[sub_agent_assistant_id]["assessment"] = (
+            response[
+                "assessment"
+            ].copy()  # Shallow copy as we don't modify individual assessments
+            if response["assessment"] is not None
+            else None
+        )

unique_toolkit-1.4.0/unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2

@@ -0,0 +1,9 @@
+Here are the sub_agent(s) assessments:
+
+{% for sub_agent in sub_agents %}
+- Agent name: {{ sub_agent.name }} 
+{% for assessment in sub_agent.assessments %}
+    - {{ assessment.title }}: {{ assessment.label }}
+        {{ assessment.explanation }}
+{% endfor %}
+{% endfor %}

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/unique_toolkit/agentic/tools/a2a/service.py

@@ -64,6 +64,7 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
             assistant_id=self.config.assistant_id,
         )
         self._subscribers: list[SubAgentResponseSubscriber] = []
+        self._should_run_evaluation = False
 
     def display_name(self) -> str:
         return self._display_name
@@ -99,7 +100,7 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
         return self.config.tool_format_information_for_user_prompt
 
     def evaluation_check_list(self) -> list[EvaluationMetricName]:
-        return []
+        return [EvaluationMetricName.SUB_AGENT] if self._should_run_evaluation else []
 
     def get_evaluation_checks_based_on_tool_response(
         self,
@@ -194,6 +195,10 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
             tool_call=tool_call,
         )
 
+        self._should_run_evaluation = (
+            response["assessment"] is not None and len(response["assessment"]) > 0
+        )
+
         self._notify_subscribers(response)
 
         if chat_id is None and self.config.reuse_chat:

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/unique_toolkit/agentic/tools/test/test_mcp_manager.py

@@ -402,8 +402,8 @@ class TestMCPManager:
         # Should only have the exclusive tool, MCP tools should be ignored
         tools = tool_manager.get_tools()
         tool_names = [tool.name for tool in tools]
-        assert "internal_search" not in tool_names
-        assert "mcp_test_tool" in tool_names
+        assert "internal_search" in tool_names
+        assert "mcp_test_tool" not in tool_names
         assert len(tools) == 1
 
     def test_init_tools_with_disabled_tool_config(

{unique_toolkit-1.3.2 → unique_toolkit-1.4.0}/unique_toolkit/agentic/tools/tool_manager.py

@@ -125,11 +125,7 @@ class ToolManager:
             if len(self._tool_choices) > 0 and t.name not in self._tool_choices:
                 continue
             # is the tool exclusive and has been choosen by the user?
-            if (
-                t.is_exclusive()
-                and len(tool_choices) > 0
-                and t.name not in tool_choices
-            ):
+            if t.is_exclusive() and len(tool_choices) > 0 and t.name in tool_choices:
                 self._tools = [t]  # override all other tools
                 break
             # if the tool is exclusive but no tool choices are given, skip it