unique_toolkit 1.4.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py

@@ -31,6 +31,9 @@ class _SubAgentToolInfo(TypedDict):
     display_name: str
 
 
+NO_ASSESSMENTS_FOUND = "NO_ASSESSMENTS_FOUND"
+
+
 class SubAgentsEvaluation(Evaluation):
     DISPLAY_NAME = "Sub Agents"
 
@@ -68,6 +71,7 @@ class SubAgentsEvaluation(Evaluation):
 
         value = ChatMessageAssessmentLabel.GREEN
 
+        # Use a dict in order to compare labels (RED being the worst)
         label_comparison_dict = defaultdict(
             lambda: 3
         )  # Unkown labels are highest in the sorting
@@ -76,33 +80,56 @@ class SubAgentsEvaluation(Evaluation):
         label_comparison_dict[ChatMessageAssessmentLabel.RED] = 0
 
         for assistant_id, tool_info in self._assistant_id_to_tool_info.items():
-            assessments = tool_info["assessment"]
-            if assessments is None or len(assessments) == 0:
-                logger.info("No assessment found for assistant %s", assistant_id)
+            assessments = tool_info["assessment"] or []
+            valid_assessments = []
+            for assessment in assessments:
+                if (
+                    assessment["label"] is None
+                    or assessment["label"] not in ChatMessageAssessmentLabel
+                ):
+                    logger.warning(
+                        "Unkown assistant label %s for assistant %s will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                if assessment["status"] != ChatMessageAssessmentStatus.DONE:
+                    logger.warning(
+                        "Assessment %s for assistant %s is not done (status: %s) will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                valid_assessments.append(assessment)
+
+            if len(valid_assessments) == 0:
+                logger.info("No valid assessment found for assistant %s", assistant_id)
                 continue
 
-            assessments_display_data = sorted(
-                assessments, key=lambda x: label_comparison_dict[x["label"]]
+            assessments = sorted(
+                valid_assessments, key=lambda x: label_comparison_dict[x["label"]]
             )
 
             for assessment in assessments:
-                if label := assessment["label"]:
-                    if label not in ChatMessageAssessmentLabel:
-                        logger.warning(
-                            "Unkown assistant label %s for assistant %s will be ignored",
-                            label,
-                            assistant_id,
-                        )
-                        continue
-                    value = min(value, label, key=lambda x: label_comparison_dict[x])
+                value = min(
+                    value, assessment["label"], key=lambda x: label_comparison_dict[x]
+                )
 
             sub_agents_display_data.append(
                 {
                     "name": tool_info["display_name"],
-                    "assessments": assessments_display_data,
+                    "assessments": assessments,
                 }
             )
 
+        if len(sub_agents_display_data) == 0:
+            logger.warning("No valid sub agent assessments found")
+            return EvaluationMetricResult(
+                name=self.get_name(),
+                value=NO_ASSESSMENTS_FOUND,
+                reason="No sub agents assessments found",
+            )
+
         should_summarize = False
         reason = ""
 
@@ -115,8 +142,6 @@ class SubAgentsEvaluation(Evaluation):
                 reason = (
                     sub_agents_display_data[0]["assessments"][0]["explanation"] or ""
                 )
-        else:
-            assert False, "No sub agents assessments found"
 
         if should_summarize:
             messages = (
@@ -148,6 +173,15 @@ class SubAgentsEvaluation(Evaluation):
     async def evaluation_metric_to_assessment(
         self, evaluation_result: EvaluationMetricResult
     ) -> EvaluationAssessmentMessage:
+        if evaluation_result.value == NO_ASSESSMENTS_FOUND:
+            return EvaluationAssessmentMessage(
+                status=ChatMessageAssessmentStatus.DONE,
+                explanation="No valid sub agents assessments found to consolidate.",
+                title=self.DISPLAY_NAME,
+                label=ChatMessageAssessmentLabel.GREEN,
+                type=self.get_assessment_type(),
+            )
+
         return EvaluationAssessmentMessage(
             status=ChatMessageAssessmentStatus.DONE,
             explanation=evaluation_result.reason,

unique_toolkit/language_model/infos.py

@@ -37,7 +37,9 @@ class LanguageModelName(StrEnum):
         "litellm:anthropic-claude-3-7-sonnet-thinking"
     )
     ANTHROPIC_CLAUDE_SONNET_4 = "litellm:anthropic-claude-sonnet-4"
+    ANTHROPIC_CLAUDE_SONNET_4_5 = "litellm:anthropic-claude-sonnet-4-5"
     ANTHROPIC_CLAUDE_OPUS_4 = "litellm:anthropic-claude-opus-4"
+    ANTHROPIC_CLAUDE_OPUS_4_1 = "litellm:anthropic-claude-opus-4-1"
     GEMINI_2_0_FLASH = "litellm:gemini-2-0-flash"
     GEMINI_2_5_FLASH = "litellm:gemini-2-5-flash"
     GEMINI_2_5_FLASH_LITE_PREVIEW_0617 = "litellm:gemini-2-5-flash-lite-preview-06-17"
@@ -663,6 +665,26 @@ class LanguageModelInfo(BaseModel):
                     info_cutoff_at=date(2025, 3, 1),
                     published_at=date(2025, 5, 1),
                 )
+            case LanguageModelName.ANTHROPIC_CLAUDE_SONNET_4_5:
+                return cls(
+                    name=model_name,
+                    capabilities=[
+                        ModelCapabilities.FUNCTION_CALLING,
+                        ModelCapabilities.STREAMING,
+                        ModelCapabilities.VISION,
+                        ModelCapabilities.REASONING,
+                    ],
+                    provider=LanguageModelProvider.LITELLM,
+                    version="claude-sonnet-4-5",
+                    encoder_name=EncoderName.O200K_BASE,  # TODO: Update encoder with litellm
+                    token_limits=LanguageModelTokenLimits(
+                        # Input limit is 200_000, we leave 20_000 tokens as buffer due to tokenizer mismatch
+                        token_limit_input=200_000,
+                        token_limit_output=64_000,
+                    ),
+                    info_cutoff_at=date(2025, 7, 1),
+                    published_at=date(2025, 9, 29),
+                )
             case LanguageModelName.ANTHROPIC_CLAUDE_OPUS_4:
                 return cls(
                     name=model_name,
@@ -683,6 +705,25 @@ class LanguageModelInfo(BaseModel):
                     info_cutoff_at=date(2025, 3, 1),
                     published_at=date(2025, 5, 1),
                 )
+            case LanguageModelName.ANTHROPIC_CLAUDE_OPUS_4_1:
+                return cls(
+                    name=model_name,
+                    capabilities=[
+                        ModelCapabilities.FUNCTION_CALLING,
+                        ModelCapabilities.STREAMING,
+                        ModelCapabilities.VISION,
+                        ModelCapabilities.REASONING,
+                    ],
+                    provider=LanguageModelProvider.LITELLM,
+                    version="claude-opus-4",
+                    encoder_name=EncoderName.O200K_BASE,  # TODO: Update encoder with litellm
+                    token_limits=LanguageModelTokenLimits(
+                        token_limit_input=200_000,
+                        token_limit_output=32_000,
+                    ),
+                    info_cutoff_at=date(2025, 3, 1),
+                    published_at=date(2025, 5, 1),
+                )
             case LanguageModelName.GEMINI_2_0_FLASH:
                 return cls(
                     name=model_name,

{unique_toolkit-1.4.0.dist-info → unique_toolkit-1.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.4.0
+Version: 1.4.2
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -118,6 +118,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.2] - 2025-09-30
+- Adding litellm models `litellm:anthropic-claude-sonnet-4-5` and `litellm:anthropic-claude-opus-4-1`
+
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
 ## [1.4.0] - 2025-09-29
 - Add ability to consolidate sub agent's assessments.
 

{unique_toolkit-1.4.0.dist-info → unique_toolkit-1.4.2.dist-info}/RECORD

@@ -54,7 +54,7 @@ unique_toolkit/agentic/tools/a2a/__init__.py,sha256=NdY0J33b1G4sbx6UWwNS74JVSAeE
 unique_toolkit/agentic/tools/a2a/config.py,sha256=exKyR-RyQ3RDJcEAKwfOdyj1flfbBaRhcdn5ROnmNB4,1513
 unique_toolkit/agentic/tools/a2a/evaluation/__init__.py,sha256=H9YhT22w8EadV9b-6IDqYqKQa41qcA3m6ADzmP7g6Cc,246
 unique_toolkit/agentic/tools/a2a/evaluation/config.py,sha256=o1Xj2H4175C1ALT8-wIfks69Xez3pgY77PFyPBYS4Hs,1692
-unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py,sha256=1yg2I72ke3BefXwvzLdS74CKVCnUXTvz7ZlQkQR9Ttw,6351
+unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py,sha256=V3y72yAZ5ynDnzp8V7UxMpnwa1Xyw7gvjPqfgHCeMkU,7660
 unique_toolkit/agentic/tools/a2a/evaluation/summarization_user_message.j2,sha256=acP1YqD_sCy6DT0V2EIfhQTmaUKeqpeWNJ7RGgceo8I,271
 unique_toolkit/agentic/tools/a2a/manager.py,sha256=yuuQuBrAcsT3gAWEdxf6EvRnL_iWtvaK14lRs21w5PA,1665
 unique_toolkit/agentic/tools/a2a/memory.py,sha256=4VFBzITCv5E_8YCc4iF4Y6FhzplS2C-FZaZHdeC7DyA,1028
@@ -123,7 +123,7 @@ unique_toolkit/language_model/__init__.py,sha256=lRQyLlbwHbNFf4-0foBU13UGb09lwEe
 unique_toolkit/language_model/builder.py,sha256=4OKfwJfj3TrgO1ezc_ewIue6W7BCQ2ZYQXUckWVPPTA,3369
 unique_toolkit/language_model/constants.py,sha256=B-topqW0r83dkC_25DeQfnPk3n53qzIHUCBS7YJ0-1U,119
 unique_toolkit/language_model/functions.py,sha256=PNCmbYovhgMSkY89p7-3DunG6jIekaZPvhh3iplG1Vg,16720
-unique_toolkit/language_model/infos.py,sha256=eHln--Y5f6znFxknV6A8m-fRaEpH5-kmRh9m-ZWqco4,57188
+unique_toolkit/language_model/infos.py,sha256=jc53AfqUyhgDRSK-nIK2S8d1RsedTNyyE_QQgGG_RFk,59256
 unique_toolkit/language_model/prompt.py,sha256=JSawaLjQg3VR-E2fK8engFyJnNdk21zaO8pPIodzN4Q,3991
 unique_toolkit/language_model/reference.py,sha256=nkX2VFz-IrUz8yqyc3G5jUMNwrNpxITBrMEKkbqqYoI,8583
 unique_toolkit/language_model/schemas.py,sha256=w23zH2OAYkTsS-wAqelUdhO9TCgis0TbFa8PszmhZYY,16501
@@ -137,7 +137,7 @@ unique_toolkit/short_term_memory/schemas.py,sha256=OhfcXyF6ACdwIXW45sKzjtZX_gkcJ
 unique_toolkit/short_term_memory/service.py,sha256=5PeVBu1ZCAfyDb2HLVvlmqSbyzBBuE9sI2o9Aajqjxg,8884
 unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/smart_rules/compile.py,sha256=cxWjb2dxEI2HGsakKdVCkSNi7VK9mr08w5sDcFCQyWI,9553
-unique_toolkit-1.4.0.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-1.4.0.dist-info/METADATA,sha256=esZSnEGEAT6jSQPUb614RktEvSQWrf5O9ZLBiStfdyM,33855
-unique_toolkit-1.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-1.4.0.dist-info/RECORD,,
+unique_toolkit-1.4.2.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-1.4.2.dist-info/METADATA,sha256=aIA-PciV6Tv8mVFR3uJFxGXqFAQ_-mtJ8-1X-he1ehQ,34076
+unique_toolkit-1.4.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-1.4.2.dist-info/RECORD,,