unique_toolkit 1.3.3__tar.gz → 1.4.1__tar.gz

{unique_toolkit-1.3.3 → unique_toolkit-1.4.1}/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
+## [1.4.0] - 2025-09-29
+- Add ability to consolidate sub agent's assessments.
+
 ## [1.3.3] - 2025-09-30
 - fix bug in exclusive tools not making them selectable
 

{unique_toolkit-1.3.3 → unique_toolkit-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.3.3
+Version: 1.4.1
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -118,6 +118,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
+## [1.4.0] - 2025-09-29
+- Add ability to consolidate sub agent's assessments.
+
 ## [1.3.3] - 2025-09-30
 - fix bug in exclusive tools not making them selectable
 

{unique_toolkit-1.3.3 → unique_toolkit-1.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "1.3.3"
+version = "1.4.1"
 description = ""
 authors = [
     "Cedric Klinkert <cedric.klinkert@unique.ch>",

{unique_toolkit-1.3.3 → unique_toolkit-1.4.1}/unique_toolkit/agentic/evaluation/schemas.py

@@ -15,6 +15,7 @@ from unique_toolkit.chat.schemas import (
 class EvaluationMetricName(StrEnum):
     HALLUCINATION = "hallucination"
     CONTEXT_RELEVANCY = "relevancy"
+    SUB_AGENT = "sub_agent"
 
 
 class EvaluationMetricInputFieldName(StrEnum):

{unique_toolkit-1.3.3 → unique_toolkit-1.4.1}/unique_toolkit/agentic/tools/a2a/config.py

@@ -23,6 +23,11 @@ class SubAgentToolDisplayConfig(BaseModel):
     remove_from_history: bool = True
 
 
+class SubAgentEvaluationConfig(BaseModel):
+    model_config = get_configuration_dict()
+    display_evalution: bool = True
+
+
 class SubAgentToolConfig(BaseToolConfig):
     model_config = get_configuration_dict()
 
@@ -43,3 +48,4 @@ class SubAgentToolConfig(BaseToolConfig):
     max_wait: float = 120.0
 
     response_display_config: SubAgentToolDisplayConfig = SubAgentToolDisplayConfig()
+    evaluation_config: SubAgentEvaluationConfig = SubAgentEvaluationConfig()

unique_toolkit-1.4.1/unique_toolkit/agentic/tools/a2a/evaluation/__init__.py

@@ -0,0 +1,6 @@
+from unique_toolkit.agentic.tools.a2a.evaluation.config import SubAgentEvaluationConfig
+from unique_toolkit.agentic.tools.a2a.evaluation.evaluator import (
+    SubAgentsEvaluation,
+)
+
+__all__ = ["SubAgentsEvaluation", "SubAgentEvaluationConfig"]

unique_toolkit-1.4.1/unique_toolkit/agentic/tools/a2a/evaluation/config.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from unique_toolkit._common.default_language_model import DEFAULT_GPT_4o
+from unique_toolkit._common.validators import LMI, get_LMI_default_field
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentType,
+)
+
+DEFAULT_EVALUATION_SYSTEM_MESSAGE_TEMPLATE = """
+You are a through and precise summarization model.
+You will receive a list of "assessments" of one or more agent(s) response(s).
+Your task is to give a brief summary (1-10 sentences) of the received assessments, following the following guidelines:
+1. You must NOT in ANY case state a fact that is not stated in the given assessments.
+2. You must focus first and foremost on the failing assessments, labeled `RED` below.
+3. You must mention each agent's name when summarizing its list of assessments.
+""".strip()
+
+with open(Path(__file__).parent / "summarization_user_message.j2", "r") as file:
+    DEFAULT_SUMMARIZATION_USER_MESSAGE_TEMPLATE = file.read().strip()
+
+
+class SubAgentEvaluationConfig(BaseModel):
+    assessment_type: ChatMessageAssessmentType = Field(
+        default=ChatMessageAssessmentType.COMPLIANCE,
+        description="The type of assessment to use in the display.",
+    )
+
+    summarization_model: LMI = get_LMI_default_field(DEFAULT_GPT_4o)
+    summarization_system_message: str = Field(
+        default=DEFAULT_EVALUATION_SYSTEM_MESSAGE_TEMPLATE,
+        description="The system message template for the summarization model.",
+    )
+    summarization_user_message_template: str = Field(
+        default=DEFAULT_SUMMARIZATION_USER_MESSAGE_TEMPLATE,
+        description="The user message template for the summarization model.",
+    )

unique_toolkit-1.4.1/unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py

@@ -0,0 +1,209 @@
+import logging
+from collections import defaultdict
+from typing import override
+
+import unique_sdk
+from jinja2 import Template
+from typing_extensions import TypedDict
+
+from unique_toolkit.agentic.evaluation.evaluation_manager import Evaluation
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.tools.a2a.evaluation.config import SubAgentEvaluationConfig
+from unique_toolkit.agentic.tools.a2a.service import SubAgentTool
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.language_model.builder import MessagesBuilder
+from unique_toolkit.language_model.schemas import LanguageModelStreamResponse
+from unique_toolkit.language_model.service import LanguageModelService
+
+logger = logging.getLogger(__name__)
+
+
+class _SubAgentToolInfo(TypedDict):
+    assessment: list[unique_sdk.Space.Assessment] | None
+    display_name: str
+
+
+NO_ASSESSMENTS_FOUND = "NO_ASSESSMENTS_FOUND"
+
+
+class SubAgentsEvaluation(Evaluation):
+    DISPLAY_NAME = "Sub Agents"
+
+    def __init__(
+        self,
+        config: SubAgentEvaluationConfig,
+        sub_agent_tools: list[SubAgentTool],
+        language_model_service: LanguageModelService,
+    ):
+        super().__init__(EvaluationMetricName.SUB_AGENT)
+        self._config = config
+
+        self._assistant_id_to_tool_info: dict[str, _SubAgentToolInfo] = {}
+        self._language_model_service = language_model_service
+
+        for sub_agent_tool in sub_agent_tools:
+            if sub_agent_tool.config.evaluation_config.display_evalution:
+                sub_agent_tool.subscribe(self)
+                self._assistant_id_to_tool_info[sub_agent_tool.config.assistant_id] = {
+                    "assessment": None,
+                    "display_name": sub_agent_tool.display_name(),
+                }
+
+    @override
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        return self._config.assessment_type
+
+    @override
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:
+        logger.info("Running sub agents evaluation")
+
+        sub_agents_display_data = []
+
+        value = ChatMessageAssessmentLabel.GREEN
+
+        # Use a dict in order to compare labels (RED being the worst)
+        label_comparison_dict = defaultdict(
+            lambda: 3
+        )  # Unkown labels are highest in the sorting
+        label_comparison_dict[ChatMessageAssessmentLabel.GREEN] = 2
+        label_comparison_dict[ChatMessageAssessmentLabel.YELLOW] = 1
+        label_comparison_dict[ChatMessageAssessmentLabel.RED] = 0
+
+        for assistant_id, tool_info in self._assistant_id_to_tool_info.items():
+            assessments = tool_info["assessment"] or []
+            valid_assessments = []
+            for assessment in assessments:
+                if (
+                    assessment["label"] is None
+                    or assessment["label"] not in ChatMessageAssessmentLabel
+                ):
+                    logger.warning(
+                        "Unkown assistant label %s for assistant %s will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                if assessment["status"] != ChatMessageAssessmentStatus.DONE:
+                    logger.warning(
+                        "Assessment %s for assistant %s is not done (status: %s) will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                valid_assessments.append(assessment)
+
+            if len(valid_assessments) == 0:
+                logger.info("No valid assessment found for assistant %s", assistant_id)
+                continue
+
+            assessments = sorted(
+                valid_assessments, key=lambda x: label_comparison_dict[x["label"]]
+            )
+
+            for assessment in assessments:
+                value = min(
+                    value, assessment["label"], key=lambda x: label_comparison_dict[x]
+                )
+
+            sub_agents_display_data.append(
+                {
+                    "name": tool_info["display_name"],
+                    "assessments": assessments,
+                }
+            )
+
+        if len(sub_agents_display_data) == 0:
+            logger.warning("No valid sub agent assessments found")
+            return EvaluationMetricResult(
+                name=self.get_name(),
+                value=NO_ASSESSMENTS_FOUND,
+                reason="No sub agents assessments found",
+            )
+
+        should_summarize = False
+        reason = ""
+
+        if len(sub_agents_display_data) > 1:
+            should_summarize = True
+        elif len(sub_agents_display_data) == 1:
+            if len(sub_agents_display_data[0]["assessments"]) > 1:
+                should_summarize = True
+            else:
+                reason = (
+                    sub_agents_display_data[0]["assessments"][0]["explanation"] or ""
+                )
+
+        if should_summarize:
+            messages = (
+                MessagesBuilder()
+                .system_message_append(self._config.summarization_system_message)
+                .user_message_append(
+                    Template(self._config.summarization_user_message_template).render(
+                        sub_agents=sub_agents_display_data,
+                    )
+                )
+                .build()
+            )
+
+            reason = await self._language_model_service.complete_async(
+                messages=messages,
+                model_name=self._config.summarization_model.name,
+                temperature=0.0,
+            )
+            reason = str(reason.choices[0].message.content)
+
+        return EvaluationMetricResult(
+            name=self.get_name(),
+            value=value,
+            reason=reason,
+            is_positive=value == ChatMessageAssessmentLabel.GREEN,
+        )
+
+    @override
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        if evaluation_result.value == NO_ASSESSMENTS_FOUND:
+            return EvaluationAssessmentMessage(
+                status=ChatMessageAssessmentStatus.DONE,
+                explanation="No valid sub agents assessments found to consolidate.",
+                title=self.DISPLAY_NAME,
+                label=ChatMessageAssessmentLabel.GREEN,
+                type=self.get_assessment_type(),
+            )
+
+        return EvaluationAssessmentMessage(
+            status=ChatMessageAssessmentStatus.DONE,
+            explanation=evaluation_result.reason,
+            title=self.DISPLAY_NAME,
+            label=evaluation_result.value,  # type: ignore
+            type=self.get_assessment_type(),
+        )
+
+    def notify_sub_agent_response(
+        self, sub_agent_assistant_id: str, response: unique_sdk.Space.Message
+    ) -> None:
+        if sub_agent_assistant_id not in self._assistant_id_to_tool_info:
+            logger.warning(
+                "Unknown assistant id %s received, assessment will be ignored.",
+                sub_agent_assistant_id,
+            )
+            return
+
+        self._assistant_id_to_tool_info[sub_agent_assistant_id]["assessment"] = (
+            response[
+                "assessment"
+            ].copy()  # Shallow copy as we don't modify individual assessments
+            if response["assessment"] is not None
+            else None
+        )

unique_toolkit-1.4.1/unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2

@@ -0,0 +1,9 @@
+Here are the sub_agent(s) assessments:
+
+{% for sub_agent in sub_agents %}
+- Agent name: {{ sub_agent.name }} 
+{% for assessment in sub_agent.assessments %}
+    - {{ assessment.title }}: {{ assessment.label }}
+        {{ assessment.explanation }}
+{% endfor %}
+{% endfor %}

{unique_toolkit-1.3.3 → unique_toolkit-1.4.1}/unique_toolkit/agentic/tools/a2a/service.py

@@ -64,6 +64,7 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
             assistant_id=self.config.assistant_id,
         )
         self._subscribers: list[SubAgentResponseSubscriber] = []
+        self._should_run_evaluation = False
 
     def display_name(self) -> str:
         return self._display_name
@@ -99,7 +100,7 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
         return self.config.tool_format_information_for_user_prompt
 
     def evaluation_check_list(self) -> list[EvaluationMetricName]:
-        return []
+        return [EvaluationMetricName.SUB_AGENT] if self._should_run_evaluation else []
 
     def get_evaluation_checks_based_on_tool_response(
         self,
@@ -194,6 +195,10 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
             tool_call=tool_call,
         )
 
+        self._should_run_evaluation = (
+            response["assessment"] is not None and len(response["assessment"]) > 0
+        )
+
         self._notify_subscribers(response)
 
         if chat_id is None and self.config.reuse_chat: