unique_toolkit 1.3.3__py3-none-any.whl → 1.4.1__py3-none-any.whl

unique_toolkit/agentic/evaluation/schemas.py

@@ -15,6 +15,7 @@ from unique_toolkit.chat.schemas import (
 class EvaluationMetricName(StrEnum):
     HALLUCINATION = "hallucination"
     CONTEXT_RELEVANCY = "relevancy"
+    SUB_AGENT = "sub_agent"
 
 
 class EvaluationMetricInputFieldName(StrEnum):

unique_toolkit/agentic/tools/a2a/config.py

@@ -23,6 +23,11 @@ class SubAgentToolDisplayConfig(BaseModel):
     remove_from_history: bool = True
 
 
+class SubAgentEvaluationConfig(BaseModel):
+    model_config = get_configuration_dict()
+    display_evalution: bool = True
+
+
 class SubAgentToolConfig(BaseToolConfig):
     model_config = get_configuration_dict()
 
@@ -43,3 +48,4 @@ class SubAgentToolConfig(BaseToolConfig):
     max_wait: float = 120.0
 
     response_display_config: SubAgentToolDisplayConfig = SubAgentToolDisplayConfig()
+    evaluation_config: SubAgentEvaluationConfig = SubAgentEvaluationConfig()

unique_toolkit/agentic/tools/a2a/evaluation/__init__.py

@@ -0,0 +1,6 @@
+from unique_toolkit.agentic.tools.a2a.evaluation.config import SubAgentEvaluationConfig
+from unique_toolkit.agentic.tools.a2a.evaluation.evaluator import (
+    SubAgentsEvaluation,
+)
+
+__all__ = ["SubAgentsEvaluation", "SubAgentEvaluationConfig"]

unique_toolkit/agentic/tools/a2a/evaluation/config.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from unique_toolkit._common.default_language_model import DEFAULT_GPT_4o
+from unique_toolkit._common.validators import LMI, get_LMI_default_field
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentType,
+)
+
+DEFAULT_EVALUATION_SYSTEM_MESSAGE_TEMPLATE = """
+You are a through and precise summarization model.
+You will receive a list of "assessments" of one or more agent(s) response(s).
+Your task is to give a brief summary (1-10 sentences) of the received assessments, following the following guidelines:
+1. You must NOT in ANY case state a fact that is not stated in the given assessments.
+2. You must focus first and foremost on the failing assessments, labeled `RED` below.
+3. You must mention each agent's name when summarizing its list of assessments.
+""".strip()
+
+with open(Path(__file__).parent / "summarization_user_message.j2", "r") as file:
+    DEFAULT_SUMMARIZATION_USER_MESSAGE_TEMPLATE = file.read().strip()
+
+
+class SubAgentEvaluationConfig(BaseModel):
+    assessment_type: ChatMessageAssessmentType = Field(
+        default=ChatMessageAssessmentType.COMPLIANCE,
+        description="The type of assessment to use in the display.",
+    )
+
+    summarization_model: LMI = get_LMI_default_field(DEFAULT_GPT_4o)
+    summarization_system_message: str = Field(
+        default=DEFAULT_EVALUATION_SYSTEM_MESSAGE_TEMPLATE,
+        description="The system message template for the summarization model.",
+    )
+    summarization_user_message_template: str = Field(
+        default=DEFAULT_SUMMARIZATION_USER_MESSAGE_TEMPLATE,
+        description="The user message template for the summarization model.",
+    )

unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py

@@ -0,0 +1,209 @@
+import logging
+from collections import defaultdict
+from typing import override
+
+import unique_sdk
+from jinja2 import Template
+from typing_extensions import TypedDict
+
+from unique_toolkit.agentic.evaluation.evaluation_manager import Evaluation
+from unique_toolkit.agentic.evaluation.schemas import (
+    EvaluationAssessmentMessage,
+    EvaluationMetricName,
+    EvaluationMetricResult,
+)
+from unique_toolkit.agentic.tools.a2a.evaluation.config import SubAgentEvaluationConfig
+from unique_toolkit.agentic.tools.a2a.service import SubAgentTool
+from unique_toolkit.chat.schemas import (
+    ChatMessageAssessmentLabel,
+    ChatMessageAssessmentStatus,
+    ChatMessageAssessmentType,
+)
+from unique_toolkit.language_model.builder import MessagesBuilder
+from unique_toolkit.language_model.schemas import LanguageModelStreamResponse
+from unique_toolkit.language_model.service import LanguageModelService
+
+logger = logging.getLogger(__name__)
+
+
+class _SubAgentToolInfo(TypedDict):
+    assessment: list[unique_sdk.Space.Assessment] | None
+    display_name: str
+
+
+NO_ASSESSMENTS_FOUND = "NO_ASSESSMENTS_FOUND"
+
+
+class SubAgentsEvaluation(Evaluation):
+    DISPLAY_NAME = "Sub Agents"
+
+    def __init__(
+        self,
+        config: SubAgentEvaluationConfig,
+        sub_agent_tools: list[SubAgentTool],
+        language_model_service: LanguageModelService,
+    ):
+        super().__init__(EvaluationMetricName.SUB_AGENT)
+        self._config = config
+
+        self._assistant_id_to_tool_info: dict[str, _SubAgentToolInfo] = {}
+        self._language_model_service = language_model_service
+
+        for sub_agent_tool in sub_agent_tools:
+            if sub_agent_tool.config.evaluation_config.display_evalution:
+                sub_agent_tool.subscribe(self)
+                self._assistant_id_to_tool_info[sub_agent_tool.config.assistant_id] = {
+                    "assessment": None,
+                    "display_name": sub_agent_tool.display_name(),
+                }
+
+    @override
+    def get_assessment_type(self) -> ChatMessageAssessmentType:
+        return self._config.assessment_type
+
+    @override
+    async def run(
+        self, loop_response: LanguageModelStreamResponse
+    ) -> EvaluationMetricResult:
+        logger.info("Running sub agents evaluation")
+
+        sub_agents_display_data = []
+
+        value = ChatMessageAssessmentLabel.GREEN
+
+        # Use a dict in order to compare labels (RED being the worst)
+        label_comparison_dict = defaultdict(
+            lambda: 3
+        )  # Unkown labels are highest in the sorting
+        label_comparison_dict[ChatMessageAssessmentLabel.GREEN] = 2
+        label_comparison_dict[ChatMessageAssessmentLabel.YELLOW] = 1
+        label_comparison_dict[ChatMessageAssessmentLabel.RED] = 0
+
+        for assistant_id, tool_info in self._assistant_id_to_tool_info.items():
+            assessments = tool_info["assessment"] or []
+            valid_assessments = []
+            for assessment in assessments:
+                if (
+                    assessment["label"] is None
+                    or assessment["label"] not in ChatMessageAssessmentLabel
+                ):
+                    logger.warning(
+                        "Unkown assistant label %s for assistant %s will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                if assessment["status"] != ChatMessageAssessmentStatus.DONE:
+                    logger.warning(
+                        "Assessment %s for assistant %s is not done (status: %s) will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                valid_assessments.append(assessment)
+
+            if len(valid_assessments) == 0:
+                logger.info("No valid assessment found for assistant %s", assistant_id)
+                continue
+
+            assessments = sorted(
+                valid_assessments, key=lambda x: label_comparison_dict[x["label"]]
+            )
+
+            for assessment in assessments:
+                value = min(
+                    value, assessment["label"], key=lambda x: label_comparison_dict[x]
+                )
+
+            sub_agents_display_data.append(
+                {
+                    "name": tool_info["display_name"],
+                    "assessments": assessments,
+                }
+            )
+
+        if len(sub_agents_display_data) == 0:
+            logger.warning("No valid sub agent assessments found")
+            return EvaluationMetricResult(
+                name=self.get_name(),
+                value=NO_ASSESSMENTS_FOUND,
+                reason="No sub agents assessments found",
+            )
+
+        should_summarize = False
+        reason = ""
+
+        if len(sub_agents_display_data) > 1:
+            should_summarize = True
+        elif len(sub_agents_display_data) == 1:
+            if len(sub_agents_display_data[0]["assessments"]) > 1:
+                should_summarize = True
+            else:
+                reason = (
+                    sub_agents_display_data[0]["assessments"][0]["explanation"] or ""
+                )
+
+        if should_summarize:
+            messages = (
+                MessagesBuilder()
+                .system_message_append(self._config.summarization_system_message)
+                .user_message_append(
+                    Template(self._config.summarization_user_message_template).render(
+                        sub_agents=sub_agents_display_data,
+                    )
+                )
+                .build()
+            )
+
+            reason = await self._language_model_service.complete_async(
+                messages=messages,
+                model_name=self._config.summarization_model.name,
+                temperature=0.0,
+            )
+            reason = str(reason.choices[0].message.content)
+
+        return EvaluationMetricResult(
+            name=self.get_name(),
+            value=value,
+            reason=reason,
+            is_positive=value == ChatMessageAssessmentLabel.GREEN,
+        )
+
+    @override
+    async def evaluation_metric_to_assessment(
+        self, evaluation_result: EvaluationMetricResult
+    ) -> EvaluationAssessmentMessage:
+        if evaluation_result.value == NO_ASSESSMENTS_FOUND:
+            return EvaluationAssessmentMessage(
+                status=ChatMessageAssessmentStatus.DONE,
+                explanation="No valid sub agents assessments found to consolidate.",
+                title=self.DISPLAY_NAME,
+                label=ChatMessageAssessmentLabel.GREEN,
+                type=self.get_assessment_type(),
+            )
+
+        return EvaluationAssessmentMessage(
+            status=ChatMessageAssessmentStatus.DONE,
+            explanation=evaluation_result.reason,
+            title=self.DISPLAY_NAME,
+            label=evaluation_result.value,  # type: ignore
+            type=self.get_assessment_type(),
+        )
+
+    def notify_sub_agent_response(
+        self, sub_agent_assistant_id: str, response: unique_sdk.Space.Message
+    ) -> None:
+        if sub_agent_assistant_id not in self._assistant_id_to_tool_info:
+            logger.warning(
+                "Unknown assistant id %s received, assessment will be ignored.",
+                sub_agent_assistant_id,
+            )
+            return
+
+        self._assistant_id_to_tool_info[sub_agent_assistant_id]["assessment"] = (
+            response[
+                "assessment"
+            ].copy()  # Shallow copy as we don't modify individual assessments
+            if response["assessment"] is not None
+            else None
+        )

unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2

@@ -0,0 +1,9 @@
+Here are the sub_agent(s) assessments:
+
+{% for sub_agent in sub_agents %}
+- Agent name: {{ sub_agent.name }} 
+{% for assessment in sub_agent.assessments %}
+    - {{ assessment.title }}: {{ assessment.label }}
+        {{ assessment.explanation }}
+{% endfor %}
+{% endfor %}

unique_toolkit/agentic/tools/a2a/service.py

@@ -64,6 +64,7 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
             assistant_id=self.config.assistant_id,
         )
         self._subscribers: list[SubAgentResponseSubscriber] = []
+        self._should_run_evaluation = False
 
     def display_name(self) -> str:
         return self._display_name
@@ -99,7 +100,7 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
         return self.config.tool_format_information_for_user_prompt
 
     def evaluation_check_list(self) -> list[EvaluationMetricName]:
-        return []
+        return [EvaluationMetricName.SUB_AGENT] if self._should_run_evaluation else []
 
     def get_evaluation_checks_based_on_tool_response(
         self,
@@ -194,6 +195,10 @@ class SubAgentTool(Tool[SubAgentToolConfig]):
             tool_call=tool_call,
         )
 
+        self._should_run_evaluation = (
+            response["assessment"] is not None and len(response["assessment"]) > 0
+        )
+
         self._notify_subscribers(response)
 
         if chat_id is None and self.config.reuse_chat:

{unique_toolkit-1.3.3.dist-info → unique_toolkit-1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.3.3
+Version: 1.4.1
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -118,6 +118,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
+## [1.4.0] - 2025-09-29
+- Add ability to consolidate sub agent's assessments.
+
 ## [1.3.3] - 2025-09-30
 - fix bug in exclusive tools not making them selectable
 

{unique_toolkit-1.3.3.dist-info → unique_toolkit-1.4.1.dist-info}/RECORD

@@ -38,7 +38,7 @@ unique_toolkit/agentic/evaluation/hallucination/prompts.py,sha256=O3Hi_rOzZlujvn
 unique_toolkit/agentic/evaluation/hallucination/service.py,sha256=Ut-f768HY4E9zEhfMoKYnGTFRZVkxWGiSTGOpgfZWYM,2447
 unique_toolkit/agentic/evaluation/hallucination/utils.py,sha256=QLsYvgAyQ5XnKEzn7ko7bXfzePD4De99TWnMKglMpds,8178
 unique_toolkit/agentic/evaluation/output_parser.py,sha256=0FDo8YY_Dc4qlTNeYyQkznzIFj9aX9wMrLOTbhhTl6g,1418
-unique_toolkit/agentic/evaluation/schemas.py,sha256=SND0s4zAyMMBpFhA-q2nJfVK83nPFQHLxr540zJKIpM,3128
+unique_toolkit/agentic/evaluation/schemas.py,sha256=m9JMCUmeqP8KhsJOVEzsz6dRXUe1uKw-bxRDtn5qwvM,3156
 unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py,sha256=4tDxHTApbaTMxN1sNS8WCqj2BweRk6YqZ5_zHP45jto,7977
 unique_toolkit/agentic/evaluation/tests/test_output_parser.py,sha256=RN_HcBbU6qy_e_PoYyUFcjWnp3ymJ6-gLj6TgEOupAI,3107
 unique_toolkit/agentic/history_manager/history_construction_with_contents.py,sha256=c8Zy3erSbHGT8AdICRRlSK91T_FN6tNpTznvUzpLbWk,9023
@@ -51,7 +51,11 @@ unique_toolkit/agentic/short_term_memory_manager/persistent_short_term_memory_ma
 unique_toolkit/agentic/thinking_manager/thinking_manager.py,sha256=41QWFsdRrbWlQHBfYCFv726UDom4WbcvaRfjCmoUOQI,4183
 unique_toolkit/agentic/tools/__init__.py,sha256=-ToY9-Xiz0K7qCUydH1h1yG6n4h1hQS8sBuSVPNEq2Y,43
 unique_toolkit/agentic/tools/a2a/__init__.py,sha256=NdY0J33b1G4sbx6UWwNS74JVSAeEtu8u_iEXOT64Uq0,187
-unique_toolkit/agentic/tools/a2a/config.py,sha256=m6INkff6jHEMB_DPm8MmtXfrGaMy1TIwXd1vFwHBFAU,1312
+unique_toolkit/agentic/tools/a2a/config.py,sha256=exKyR-RyQ3RDJcEAKwfOdyj1flfbBaRhcdn5ROnmNB4,1513
+unique_toolkit/agentic/tools/a2a/evaluation/__init__.py,sha256=H9YhT22w8EadV9b-6IDqYqKQa41qcA3m6ADzmP7g6Cc,246
+unique_toolkit/agentic/tools/a2a/evaluation/config.py,sha256=o1Xj2H4175C1ALT8-wIfks69Xez3pgY77PFyPBYS4Hs,1692
+unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py,sha256=V3y72yAZ5ynDnzp8V7UxMpnwa1Xyw7gvjPqfgHCeMkU,7660
+unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2,sha256=acP1YqD_sCy6DT0V2EIfhQTmaUKeqpeWNJ7RGgceo8I,271
 unique_toolkit/agentic/tools/a2a/manager.py,sha256=yuuQuBrAcsT3gAWEdxf6EvRnL_iWtvaK14lRs21w5PA,1665
 unique_toolkit/agentic/tools/a2a/memory.py,sha256=4VFBzITCv5E_8YCc4iF4Y6FhzplS2C-FZaZHdeC7DyA,1028
 unique_toolkit/agentic/tools/a2a/postprocessing/__init__.py,sha256=9M5RRSO4gmQVI1gfve6MGEfVktT9WcELQFZ8Sv6xu4Y,160
@@ -59,7 +63,7 @@ unique_toolkit/agentic/tools/a2a/postprocessing/display.py,sha256=rlBWO2M8Lr5Kx-
 unique_toolkit/agentic/tools/a2a/postprocessing/postprocessor.py,sha256=BWXFsv5akPhkNxRQ3JLktrBkoT1_66joTel_uM4BgRY,6963
 unique_toolkit/agentic/tools/a2a/postprocessing/test/test_display.py,sha256=UVOd5tVTltngVfsdOn6cuyvG7QmRBkfgUj_SruTgmHI,15279
 unique_toolkit/agentic/tools/a2a/schema.py,sha256=T1l5z6trtPE5nhqPzt5tvfRNDhqL_ST1Wj7_lBWJ58g,304
-unique_toolkit/agentic/tools/a2a/service.py,sha256=cACUPwWXzi_XfPlFqK0x6nawugx17Vq18-XRUbhgL4k,7669
+unique_toolkit/agentic/tools/a2a/service.py,sha256=ySgCZZ_3APPZ8j0sz-7_1CYxfFTvGCSpu0qsUG1YShQ,7916
 unique_toolkit/agentic/tools/agent_chunks_hanlder.py,sha256=x32Dp1Z8cVW5i-XzXbaMwX2KHPcNGmqEU-FB4AV9ZGo,1909
 unique_toolkit/agentic/tools/config.py,sha256=91Gw92YoTC6WeWa4lfOpXSvIYekCwELcVNkHZZEkW2o,4936
 unique_toolkit/agentic/tools/factory.py,sha256=Wt0IGSbLg8ZTq5PU9p_JTW0LtNATWLpc3336irJKXlM,1277
@@ -133,7 +137,7 @@ unique_toolkit/short_term_memory/schemas.py,sha256=OhfcXyF6ACdwIXW45sKzjtZX_gkcJ
 unique_toolkit/short_term_memory/service.py,sha256=5PeVBu1ZCAfyDb2HLVvlmqSbyzBBuE9sI2o9Aajqjxg,8884
 unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/smart_rules/compile.py,sha256=cxWjb2dxEI2HGsakKdVCkSNi7VK9mr08w5sDcFCQyWI,9553
-unique_toolkit-1.3.3.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-1.3.3.dist-info/METADATA,sha256=FY2iBzcDa6mneFLi2qOfuttrXjCBDkW1TM11UoBbGus,33776
-unique_toolkit-1.3.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-1.3.3.dist-info/RECORD,,
+unique_toolkit-1.4.1.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-1.4.1.dist-info/METADATA,sha256=XwdO5BNQidG6zSM_lKxs4SZQYeALlNREgBWrygFFw7o,33949
+unique_toolkit-1.4.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-1.4.1.dist-info/RECORD,,