unique_toolkit 1.4.0__py3-none-any.whl → 1.4.1__py3-none-any.whl

unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py

@@ -31,6 +31,9 @@ class _SubAgentToolInfo(TypedDict):
     display_name: str
 
 
+NO_ASSESSMENTS_FOUND = "NO_ASSESSMENTS_FOUND"
+
+
 class SubAgentsEvaluation(Evaluation):
     DISPLAY_NAME = "Sub Agents"
 
@@ -68,6 +71,7 @@ class SubAgentsEvaluation(Evaluation):
 
         value = ChatMessageAssessmentLabel.GREEN
 
+        # Use a dict in order to compare labels (RED being the worst)
         label_comparison_dict = defaultdict(
             lambda: 3
         )  # Unkown labels are highest in the sorting
@@ -76,33 +80,56 @@ class SubAgentsEvaluation(Evaluation):
         label_comparison_dict[ChatMessageAssessmentLabel.RED] = 0
 
         for assistant_id, tool_info in self._assistant_id_to_tool_info.items():
-            assessments = tool_info["assessment"]
-            if assessments is None or len(assessments) == 0:
-                logger.info("No assessment found for assistant %s", assistant_id)
+            assessments = tool_info["assessment"] or []
+            valid_assessments = []
+            for assessment in assessments:
+                if (
+                    assessment["label"] is None
+                    or assessment["label"] not in ChatMessageAssessmentLabel
+                ):
+                    logger.warning(
+                        "Unkown assistant label %s for assistant %s will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                if assessment["status"] != ChatMessageAssessmentStatus.DONE:
+                    logger.warning(
+                        "Assessment %s for assistant %s is not done (status: %s) will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                valid_assessments.append(assessment)
+
+            if len(valid_assessments) == 0:
+                logger.info("No valid assessment found for assistant %s", assistant_id)
                 continue
 
-            assessments_display_data = sorted(
-                assessments, key=lambda x: label_comparison_dict[x["label"]]
+            assessments = sorted(
+                valid_assessments, key=lambda x: label_comparison_dict[x["label"]]
             )
 
             for assessment in assessments:
-                if label := assessment["label"]:
-                    if label not in ChatMessageAssessmentLabel:
-                        logger.warning(
-                            "Unkown assistant label %s for assistant %s will be ignored",
-                            label,
-                            assistant_id,
-                        )
-                        continue
-                    value = min(value, label, key=lambda x: label_comparison_dict[x])
+                value = min(
+                    value, assessment["label"], key=lambda x: label_comparison_dict[x]
+                )
 
             sub_agents_display_data.append(
                 {
                     "name": tool_info["display_name"],
-                    "assessments": assessments_display_data,
+                    "assessments": assessments,
                 }
             )
 
+        if len(sub_agents_display_data) == 0:
+            logger.warning("No valid sub agent assessments found")
+            return EvaluationMetricResult(
+                name=self.get_name(),
+                value=NO_ASSESSMENTS_FOUND,
+                reason="No sub agents assessments found",
+            )
+
         should_summarize = False
         reason = ""
 
@@ -115,8 +142,6 @@ class SubAgentsEvaluation(Evaluation):
                 reason = (
                     sub_agents_display_data[0]["assessments"][0]["explanation"] or ""
                 )
-        else:
-            assert False, "No sub agents assessments found"
 
         if should_summarize:
             messages = (
@@ -148,6 +173,15 @@ class SubAgentsEvaluation(Evaluation):
     async def evaluation_metric_to_assessment(
         self, evaluation_result: EvaluationMetricResult
     ) -> EvaluationAssessmentMessage:
+        if evaluation_result.value == NO_ASSESSMENTS_FOUND:
+            return EvaluationAssessmentMessage(
+                status=ChatMessageAssessmentStatus.DONE,
+                explanation="No valid sub agents assessments found to consolidate.",
+                title=self.DISPLAY_NAME,
+                label=ChatMessageAssessmentLabel.GREEN,
+                type=self.get_assessment_type(),
+            )
+
         return EvaluationAssessmentMessage(
             status=ChatMessageAssessmentStatus.DONE,
             explanation=evaluation_result.reason,

{unique_toolkit-1.4.0.dist-info → unique_toolkit-1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.4.0
+Version: 1.4.1
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -118,6 +118,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
 ## [1.4.0] - 2025-09-29
 - Add ability to consolidate sub agent's assessments.
 

{unique_toolkit-1.4.0.dist-info → unique_toolkit-1.4.1.dist-info}/RECORD

@@ -54,7 +54,7 @@ unique_toolkit/agentic/tools/a2a/__init__.py,sha256=NdY0J33b1G4sbx6UWwNS74JVSAeE
 unique_toolkit/agentic/tools/a2a/config.py,sha256=exKyR-RyQ3RDJcEAKwfOdyj1flfbBaRhcdn5ROnmNB4,1513
 unique_toolkit/agentic/tools/a2a/evaluation/__init__.py,sha256=H9YhT22w8EadV9b-6IDqYqKQa41qcA3m6ADzmP7g6Cc,246
 unique_toolkit/agentic/tools/a2a/evaluation/config.py,sha256=o1Xj2H4175C1ALT8-wIfks69Xez3pgY77PFyPBYS4Hs,1692
-unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py,sha256=1yg2I72ke3BefXwvzLdS74CKVCnUXTvz7ZlQkQR9Ttw,6351
+unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py,sha256=V3y72yAZ5ynDnzp8V7UxMpnwa1Xyw7gvjPqfgHCeMkU,7660
 unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2,sha256=acP1YqD_sCy6DT0V2EIfhQTmaUKeqpeWNJ7RGgceo8I,271
 unique_toolkit/agentic/tools/a2a/manager.py,sha256=yuuQuBrAcsT3gAWEdxf6EvRnL_iWtvaK14lRs21w5PA,1665
 unique_toolkit/agentic/tools/a2a/memory.py,sha256=4VFBzITCv5E_8YCc4iF4Y6FhzplS2C-FZaZHdeC7DyA,1028
@@ -137,7 +137,7 @@ unique_toolkit/short_term_memory/schemas.py,sha256=OhfcXyF6ACdwIXW45sKzjtZX_gkcJ
 unique_toolkit/short_term_memory/service.py,sha256=5PeVBu1ZCAfyDb2HLVvlmqSbyzBBuE9sI2o9Aajqjxg,8884
 unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/smart_rules/compile.py,sha256=cxWjb2dxEI2HGsakKdVCkSNi7VK9mr08w5sDcFCQyWI,9553
-unique_toolkit-1.4.0.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-1.4.0.dist-info/METADATA,sha256=esZSnEGEAT6jSQPUb614RktEvSQWrf5O9ZLBiStfdyM,33855
-unique_toolkit-1.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-1.4.0.dist-info/RECORD,,
+unique_toolkit-1.4.1.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-1.4.1.dist-info/METADATA,sha256=XwdO5BNQidG6zSM_lKxs4SZQYeALlNREgBWrygFFw7o,33949
+unique_toolkit-1.4.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-1.4.1.dist-info/RECORD,,