unique_toolkit 1.4.0__tar.gz → 1.4.2__tar.gz

{unique_toolkit-1.4.0 → unique_toolkit-1.4.2}/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.2] - 2025-09-30
+- Adding litellm models `litellm:anthropic-claude-sonnet-4-5` and `litellm:anthropic-claude-opus-4-1`
+
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
 ## [1.4.0] - 2025-09-29
 - Add ability to consolidate sub agent's assessments.
 

{unique_toolkit-1.4.0 → unique_toolkit-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.4.0
+Version: 1.4.2
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -118,6 +118,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.2] - 2025-09-30
+- Adding litellm models `litellm:anthropic-claude-sonnet-4-5` and `litellm:anthropic-claude-opus-4-1`
+
+## [1.4.1] - 2025-09-30
+- Handle sub agent failed assessments better in sub agent evaluator.
+
 ## [1.4.0] - 2025-09-29
 - Add ability to consolidate sub agent's assessments.
 

{unique_toolkit-1.4.0 → unique_toolkit-1.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "1.4.0"
+version = "1.4.2"
 description = ""
 authors = [
     "Cedric Klinkert <cedric.klinkert@unique.ch>",

{unique_toolkit-1.4.0 → unique_toolkit-1.4.2}/unique_toolkit/agentic/tools/a2a/evaluation/evaluator.py

@@ -31,6 +31,9 @@ class _SubAgentToolInfo(TypedDict):
     display_name: str
 
 
+NO_ASSESSMENTS_FOUND = "NO_ASSESSMENTS_FOUND"
+
+
 class SubAgentsEvaluation(Evaluation):
     DISPLAY_NAME = "Sub Agents"
 
@@ -68,6 +71,7 @@ class SubAgentsEvaluation(Evaluation):
 
         value = ChatMessageAssessmentLabel.GREEN
 
+        # Use a dict in order to compare labels (RED being the worst)
         label_comparison_dict = defaultdict(
             lambda: 3
         )  # Unkown labels are highest in the sorting
@@ -76,33 +80,56 @@ class SubAgentsEvaluation(Evaluation):
         label_comparison_dict[ChatMessageAssessmentLabel.RED] = 0
 
         for assistant_id, tool_info in self._assistant_id_to_tool_info.items():
-            assessments = tool_info["assessment"]
-            if assessments is None or len(assessments) == 0:
-                logger.info("No assessment found for assistant %s", assistant_id)
+            assessments = tool_info["assessment"] or []
+            valid_assessments = []
+            for assessment in assessments:
+                if (
+                    assessment["label"] is None
+                    or assessment["label"] not in ChatMessageAssessmentLabel
+                ):
+                    logger.warning(
+                        "Unkown assistant label %s for assistant %s will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                if assessment["status"] != ChatMessageAssessmentStatus.DONE:
+                    logger.warning(
+                        "Assessment %s for assistant %s is not done (status: %s) will be ignored",
+                        assessment["label"],
+                        assistant_id,
+                    )
+                    continue
+                valid_assessments.append(assessment)
+
+            if len(valid_assessments) == 0:
+                logger.info("No valid assessment found for assistant %s", assistant_id)
                 continue
 
-            assessments_display_data = sorted(
-                assessments, key=lambda x: label_comparison_dict[x["label"]]
+            assessments = sorted(
+                valid_assessments, key=lambda x: label_comparison_dict[x["label"]]
             )
 
             for assessment in assessments:
-                if label := assessment["label"]:
-                    if label not in ChatMessageAssessmentLabel:
-                        logger.warning(
-                            "Unkown assistant label %s for assistant %s will be ignored",
-                            label,
-                            assistant_id,
-                        )
-                        continue
-                    value = min(value, label, key=lambda x: label_comparison_dict[x])
+                value = min(
+                    value, assessment["label"], key=lambda x: label_comparison_dict[x]
+                )
 
             sub_agents_display_data.append(
                 {
                     "name": tool_info["display_name"],
-                    "assessments": assessments_display_data,
+                    "assessments": assessments,
                 }
             )
 
+        if len(sub_agents_display_data) == 0:
+            logger.warning("No valid sub agent assessments found")
+            return EvaluationMetricResult(
+                name=self.get_name(),
+                value=NO_ASSESSMENTS_FOUND,
+                reason="No sub agents assessments found",
+            )
+
         should_summarize = False
         reason = ""
 
@@ -115,8 +142,6 @@ class SubAgentsEvaluation(Evaluation):
                 reason = (
                     sub_agents_display_data[0]["assessments"][0]["explanation"] or ""
                 )
-        else:
-            assert False, "No sub agents assessments found"
 
         if should_summarize:
             messages = (
@@ -148,6 +173,15 @@ class SubAgentsEvaluation(Evaluation):
     async def evaluation_metric_to_assessment(
         self, evaluation_result: EvaluationMetricResult
     ) -> EvaluationAssessmentMessage:
+        if evaluation_result.value == NO_ASSESSMENTS_FOUND:
+            return EvaluationAssessmentMessage(
+                status=ChatMessageAssessmentStatus.DONE,
+                explanation="No valid sub agents assessments found to consolidate.",
+                title=self.DISPLAY_NAME,
+                label=ChatMessageAssessmentLabel.GREEN,
+                type=self.get_assessment_type(),
+            )
+
         return EvaluationAssessmentMessage(
             status=ChatMessageAssessmentStatus.DONE,
             explanation=evaluation_result.reason,

{unique_toolkit-1.4.0 → unique_toolkit-1.4.2}/unique_toolkit/language_model/infos.py

@@ -37,7 +37,9 @@ class LanguageModelName(StrEnum):
         "litellm:anthropic-claude-3-7-sonnet-thinking"
     )
     ANTHROPIC_CLAUDE_SONNET_4 = "litellm:anthropic-claude-sonnet-4"
+    ANTHROPIC_CLAUDE_SONNET_4_5 = "litellm:anthropic-claude-sonnet-4-5"
     ANTHROPIC_CLAUDE_OPUS_4 = "litellm:anthropic-claude-opus-4"
+    ANTHROPIC_CLAUDE_OPUS_4_1 = "litellm:anthropic-claude-opus-4-1"
     GEMINI_2_0_FLASH = "litellm:gemini-2-0-flash"
     GEMINI_2_5_FLASH = "litellm:gemini-2-5-flash"
     GEMINI_2_5_FLASH_LITE_PREVIEW_0617 = "litellm:gemini-2-5-flash-lite-preview-06-17"
@@ -663,6 +665,26 @@ class LanguageModelInfo(BaseModel):
                     info_cutoff_at=date(2025, 3, 1),
                     published_at=date(2025, 5, 1),
                 )
+            case LanguageModelName.ANTHROPIC_CLAUDE_SONNET_4_5:
+                return cls(
+                    name=model_name,
+                    capabilities=[
+                        ModelCapabilities.FUNCTION_CALLING,
+                        ModelCapabilities.STREAMING,
+                        ModelCapabilities.VISION,
+                        ModelCapabilities.REASONING,
+                    ],
+                    provider=LanguageModelProvider.LITELLM,
+                    version="claude-sonnet-4-5",
+                    encoder_name=EncoderName.O200K_BASE,  # TODO: Update encoder with litellm
+                    token_limits=LanguageModelTokenLimits(
+                        # Input limit is 200_000, we leave 20_000 tokens as buffer due to tokenizer mismatch
+                        token_limit_input=200_000,
+                        token_limit_output=64_000,
+                    ),
+                    info_cutoff_at=date(2025, 7, 1),
+                    published_at=date(2025, 9, 29),
+                )
             case LanguageModelName.ANTHROPIC_CLAUDE_OPUS_4:
                 return cls(
                     name=model_name,
@@ -683,6 +705,25 @@ class LanguageModelInfo(BaseModel):
                     info_cutoff_at=date(2025, 3, 1),
                     published_at=date(2025, 5, 1),
                 )
+            case LanguageModelName.ANTHROPIC_CLAUDE_OPUS_4_1:
+                return cls(
+                    name=model_name,
+                    capabilities=[
+                        ModelCapabilities.FUNCTION_CALLING,
+                        ModelCapabilities.STREAMING,
+                        ModelCapabilities.VISION,
+                        ModelCapabilities.REASONING,
+                    ],
+                    provider=LanguageModelProvider.LITELLM,
+                    version="claude-opus-4",
+                    encoder_name=EncoderName.O200K_BASE,  # TODO: Update encoder with litellm
+                    token_limits=LanguageModelTokenLimits(
+                        token_limit_input=200_000,
+                        token_limit_output=32_000,
+                    ),
+                    info_cutoff_at=date(2025, 3, 1),
+                    published_at=date(2025, 5, 1),
+                )
             case LanguageModelName.GEMINI_2_0_FLASH:
                 return cls(
                     name=model_name,