unique_toolkit 2026.30.0__tar.gz → 2026.30.0.dev0__tar.gz

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 2026.30.0
+Version: 2026.30.0.dev0
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
 Requires-Dist: openai>=1.105.0,<3
 Requires-Dist: platformdirs>=4.0.0,<5
 Requires-Dist: pillow>=12.1.1
-Requires-Dist: unique-sdk>=2026.30.0
+Requires-Dist: unique-sdk>=2026.30.0.dev0,<2026.30.0rc0
 Requires-Dist: jambo>=0.1.2,<0.2
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: docxtpl>=0.20.1,<0.21
@@ -32,7 +32,6 @@ Requires-Dist: pypandoc>=1.16.2,<2
 Requires-Dist: tenacity>=9.1.4
 Requires-Dist: pydantic-core>=2.41.5
 Requires-Dist: cachetools>=5.0,<6
-Requires-Dist: rapidfuzz>=3.0,<4
 Requires-Dist: fastapi>=0.121.0,<0.122 ; extra == 'fastapi'
 Requires-Dist: starlette>=0.49.1 ; extra == 'fastapi'
 Requires-Dist: uvicorn>=0.37.0,<0.38 ; extra == 'fastapi'

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "2026.30.0"
+version = "2026.30.0.dev0"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,7 +28,7 @@ dependencies = [
     "openai>=1.105.0,<3",
     "platformdirs>=4.0.0,<5",
     "pillow>=12.1.1",
-    "unique-sdk>=2026.30.0",
+    "unique-sdk>=2026.30.0.dev0,<2026.30.0rc0",
     "jambo>=0.1.2,<0.2",
     "jinja2>=3.1.6,<4",
     "docxtpl>=0.20.1,<0.21",
@@ -44,7 +44,6 @@ dependencies = [
     "tenacity>=9.1.4",
     "pydantic-core>=2.41.5",
     "cachetools>=5.0,<6",
-    "rapidfuzz>=3.0,<4",
 ]
 
 [project.optional-dependencies]

unique_toolkit-2026.30.0.dev0/unique_toolkit/agentic/debug_info_manager/debug_info_manager.py

@@ -0,0 +1,92 @@
+from typing import Any
+
+from unique_toolkit.agentic.tools.openai_builtin import OpenAICodeInterpreterTool
+from unique_toolkit.agentic.tools.openai_builtin.base import OpenAIBuiltInToolName
+from unique_toolkit.agentic.tools.schemas import ToolCallResponse
+from unique_toolkit.agentic.tools.tool_manager import ToolManager
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+    ResponsesLanguageModelStreamResponse,
+)
+
+
+class DebugInfoManager:
+    def __init__(self):
+        self.debug_info = {"tools": []}
+
+    def extract_tool_debug_info(
+        self,
+        tool_call_responses: list[ToolCallResponse],
+        loop_iteration_index: int | None = None,
+    ) -> None:
+        for tool_call_response in tool_call_responses:
+            debug_info = (
+                tool_call_response.debug_info.copy()
+                if tool_call_response.debug_info
+                else {}
+            )
+            tool_info: dict[str, Any] = {
+                "name": tool_call_response.name,
+                "info": debug_info,
+            }
+            if debug_info.get("mcp_server"):
+                tool_info["mcp_server"] = debug_info["mcp_server"]
+            if loop_iteration_index is not None:
+                tool_info["info"]["loop_iteration"] = loop_iteration_index
+            self.debug_info["tools"].append(tool_info)
+
+    def extract_builtin_tool_debug_info(
+        self,
+        stream_response: LanguageModelStreamResponse,
+        tool_manager: ToolManager,
+        loop_iteration_index: int | None = None,
+    ) -> None:
+        self.debug_info["tools"].extend(
+            _extract_tool_calls_from_stream_response(
+                stream_response, tool_manager, loop_iteration_index
+            )
+        )
+
+    def add(self, key: str, value: Any) -> None:
+        self.debug_info = self.debug_info | {key: value}
+
+    def get(self) -> dict[str, Any]:
+        return self.debug_info
+
+
+def _extract_tool_calls_from_stream_response(
+    stream_response: LanguageModelStreamResponse,
+    tool_manager: ToolManager,
+    loop_iteration_index: int | None = None,
+) -> list[dict[str, Any]]:
+    if not isinstance(stream_response, ResponsesLanguageModelStreamResponse):
+        return []
+
+    seen = set()
+    tool_infos = []
+
+    for code_interpreter_call in stream_response.code_interpreter_calls:
+        if code_interpreter_call.id in seen:
+            continue
+
+        seen.add(code_interpreter_call.id)
+        tool_name = OpenAIBuiltInToolName.CODE_INTERPRETER
+
+        is_exclusive = tool_name in tool_manager.get_exclusive_tools()
+        is_forced = tool_name in tool_manager.get_tool_choices()
+
+        debug_info = OpenAICodeInterpreterTool.get_debug_info(code_interpreter_call)
+
+        if loop_iteration_index is not None:
+            debug_info["loop_iteration"] = loop_iteration_index
+        debug_info["is_exclusive"] = is_exclusive
+        debug_info["is_forced"] = is_forced
+
+        tool_infos.append(
+            {
+                "name": tool_name,
+                "info": debug_info,
+            }
+        )
+
+    return tool_infos

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/unique_toolkit/agentic/evaluation/context_relevancy/service.py

@@ -32,9 +32,6 @@ from unique_toolkit.language_model.infos import (
     LanguageModelInfo,
     ModelCapabilities,
 )
-from unique_toolkit.language_model.invocation_stats import (
-    LanguageModelInvocationStats,
-)
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessages,
     LanguageModelSystemMessage,
@@ -148,10 +145,6 @@ class ContextRelevancyEvaluator:
             # Handle regular output
             return await self._handle_regular_output(input, config)
 
-        except EvaluatorException:
-            # Already carries whatever invocation_stats were captured before
-            # the failure -- don't re-wrap it and lose them.
-            raise
         except Exception as e:
             error_message = (
                 "Unknown error occurred during context relevancy metric analysis"
@@ -179,49 +172,21 @@ class ContextRelevancyEvaluator:
             other_options=config.additional_llm_options,
         )
 
-        # Captured as soon as the (billable) LLM call returns, so a validation
-        # failure below can still report it instead of losing it.
-        invocation_stats = (
-            [
-                LanguageModelInvocationStats.from_usage(
-                    config.language_model.name,
-                    result.usage,
-                    source="context_relevancy",
-                )
-            ]
-            if result.usage is not None
-            else []
-        )
-
         try:
             result_content = EvaluationSchemaStructuredOutput.model_validate(
                 result.choices[0].message.parsed
             )
-            evaluation_result = parse_eval_metric_result_structured_output(
-                result_content, EvaluationMetricName.CONTEXT_RELEVANCY
-            )
         except ValidationError as e:
             error_message = "Error occurred during structured output validation of the context relevancy evaluation."
             raise EvaluatorException(
                 error_message=error_message,
                 user_message=error_message,
                 exception=e,
-                invocation_stats=invocation_stats,
-            )
-        except Exception as e:
-            # Catches anything else reading the response (e.g. an empty
-            # `choices` list) -- must still carry invocation_stats, not just
-            # the validation-specific failure above.
-            error_message = "Error occurred while reading the structured output response for context relevancy evaluation."
-            raise EvaluatorException(
-                error_message=f"{error_message}: {e}",
-                user_message=error_message,
-                exception=e,
-                invocation_stats=invocation_stats,
             )
 
-        evaluation_result.invocation_stats = invocation_stats
-        return evaluation_result
+        return parse_eval_metric_result_structured_output(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
 
     async def _handle_regular_output(
         self,
@@ -236,52 +201,17 @@ class ContextRelevancyEvaluator:
             other_options=config.additional_llm_options,
         )
 
-        # Captured as soon as the (billable) LLM call returns, so a failure
-        # below can still report it instead of losing it.
-        invocation_stats = (
-            [
-                LanguageModelInvocationStats.from_usage(
-                    config.language_model.name,
-                    result.usage,
-                    source="context_relevancy",
-                )
-            ]
-            if result.usage is not None
-            else []
-        )
-
-        try:
-            result_content = result.choices[0].message.content
-            if not result_content or not isinstance(result_content, str):
-                error_message = "Context relevancy evaluation did not return a result."
-                raise EvaluatorException(
-                    error_message=error_message,
-                    user_message=error_message,
-                    invocation_stats=invocation_stats,
-                )
-            evaluation_result = parse_eval_metric_result(
-                result_content, EvaluationMetricName.CONTEXT_RELEVANCY
-            )
-        except EvaluatorException as e:
-            # Covers both the raise above and parse_eval_metric_result's own
-            # EvaluatorException (which has no invocation_stats, since it has
-            # no usage context) -- attach ours if it didn't already carry
-            # stats, instead of letting it propagate stats-less.
-            if not e.invocation_stats:
-                e.invocation_stats = invocation_stats
-            raise
-        except Exception as e:
-            # Catches anything else reading the response (e.g. an empty
-            # `choices` list).
-            error_message = "Error occurred while reading the context relevancy evaluation response."
+        result_content = result.choices[0].message.content
+        if not result_content or not isinstance(result_content, str):
+            error_message = "Context relevancy evaluation did not return a result."
             raise EvaluatorException(
-                error_message=f"{error_message}: {e}",
+                error_message=error_message,
                 user_message=error_message,
-                exception=e,
-                invocation_stats=invocation_stats,
             )
-        evaluation_result.invocation_stats = invocation_stats
-        return evaluation_result
+
+        return parse_eval_metric_result(
+            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+        )
 
     def _compose_msgs(
         self,

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/unique_toolkit/agentic/evaluation/evaluation_manager.py

@@ -7,7 +7,6 @@ from unique_toolkit._common.execution import (
     Result,
     SafeTaskExecutor,
 )
-from unique_toolkit.agentic.evaluation.exception import EvaluatorException
 from unique_toolkit.agentic.evaluation.schemas import (
     EvaluationAssessmentMessage,
     EvaluationMetricName,
@@ -18,9 +17,6 @@ from unique_toolkit.chat.schemas import (
     ChatMessageAssessmentType,
 )
 from unique_toolkit.chat.service import ChatService
-from unique_toolkit.language_model.invocation_stats import (
-    LanguageModelInvocationStats,
-)
 from unique_toolkit.language_model.schemas import (
     LanguageModelStreamResponse,
 )
@@ -92,7 +88,6 @@ class EvaluationManager:
         self._evaluations: dict[EvaluationMetricName, Evaluation] = {}
         self._evaluation_passed: bool = True
         self._execution_times: dict[str, float] = {}
-        self._invocation_stats: list[LanguageModelInvocationStats] = []
 
     def add_evaluation(self, evaluation: Evaluation):
         self._evaluations[evaluation.get_name()] = evaluation
@@ -107,7 +102,6 @@ class EvaluationManager:
         assistant_message_id: str,
     ) -> list[EvaluationMetricResult]:
         self._execution_times = {}
-        self._invocation_stats = []
 
         task_executor = SafeTaskExecutor(
             logger=self._logger,
@@ -131,7 +125,6 @@ class EvaluationManager:
             )
             if not unpacked_evaluation_result.is_positive:
                 self._evaluation_passed = False
-            self._invocation_stats.extend(unpacked_evaluation_result.invocation_stats)
             evaluation_results_unpacked.append(unpacked_evaluation_result)
 
         for evaluation_name, evaluation_result in zip(
@@ -148,10 +141,6 @@ class EvaluationManager:
     def get_execution_times(self) -> dict[str, float]:
         return self._execution_times.copy()
 
-    def get_invocation_stats(self) -> list[LanguageModelInvocationStats]:
-        """Per-LLM-call stats from every evaluation that made its own LLM call."""
-        return list(self._invocation_stats)
-
     async def execute_evaluation_call(
         self,
         evaluation_name: EvaluationMetricName,
@@ -189,18 +178,12 @@ class EvaluationManager:
         evaluation_name: EvaluationMetricName,
     ) -> EvaluationMetricResult:
         if not result.success:
-            invocation_stats = (
-                result.exception.invocation_stats
-                if isinstance(result.exception, EvaluatorException)
-                else []
-            )
             return EvaluationMetricResult(
                 name=evaluation_name,
                 is_positive=True,
                 value="RED",
                 reason=str(result.exception),
                 error=Exception("Evaluation result is not successful"),
-                invocation_stats=invocation_stats,
             )
         return result.unpack()
 

unique_toolkit-2026.30.0.dev0/unique_toolkit/agentic/evaluation/exception.py

@@ -0,0 +1,5 @@
+from unique_toolkit._common.exception import CommonException
+
+
+class EvaluatorException(CommonException):
+    pass

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/unique_toolkit/agentic/evaluation/hallucination/utils.py

@@ -18,9 +18,6 @@ from unique_toolkit.content import ContentReference
 from unique_toolkit.content.schemas import ContentChunk
 from unique_toolkit.language_model.builder import MessagesBuilder
 from unique_toolkit.language_model.infos import ModelCapabilities
-from unique_toolkit.language_model.invocation_stats import (
-    LanguageModelInvocationStats,
-)
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessages,
     LanguageModelStreamResponse,
@@ -76,58 +73,29 @@ async def check_hallucination(
 
     try:
         msgs = _get_msgs(input, config)
+
         result = await LanguageModelService.complete_async_util(
             company_id=company_id, user_id=user_id, messages=msgs, model_name=model_name
         )
-    except Exception as e:
-        error_message = "Error occurred during hallucination metric analysis"
-        raise EvaluatorException(
-            error_message=f"{error_message}: {e}",
-            user_message=error_message,
-            exception=e,
-        )
-
-    # Captured as soon as the (billable) LLM call returns, so a failure below
-    # (empty content, parsing) can still report it instead of losing it.
-    invocation_stats = (
-        [
-            LanguageModelInvocationStats.from_usage(
-                model_name, result.usage, source="hallucination"
-            )
-        ]
-        if result.usage is not None
-        else []
-    )
-
-    try:
         result_content = result.choices[0].message.content
         if not result_content or not isinstance(result_content, str):
             error_message = "Hallucination evaluation did not return a text result."
             raise EvaluatorException(
                 error_message=error_message,
                 user_message=error_message,
-                invocation_stats=invocation_stats,
             )
-        eval_result = parse_eval_metric_result(
+        result = parse_eval_metric_result(
             result_content,
             EvaluationMetricName.HALLUCINATION,
         )
-        eval_result.invocation_stats = invocation_stats
-        return eval_result
-    except EvaluatorException as e:
-        # parse_eval_metric_result raises its own EvaluatorException (with no
-        # invocation_stats, since it has no usage context) -- attach ours if
-        # it didn't already carry stats, instead of re-raising it stats-less.
-        if not e.invocation_stats:
-            e.invocation_stats = invocation_stats
-        raise
+
+        return result
     except Exception as e:
         error_message = "Error occurred during hallucination metric analysis"
         raise EvaluatorException(
             error_message=f"{error_message}: {e}",
             user_message=error_message,
             exception=e,
-            invocation_stats=invocation_stats,
         )
 
 

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/unique_toolkit/agentic/evaluation/schemas.py

@@ -10,7 +10,6 @@ from unique_toolkit.chat.schemas import (
     ChatMessageAssessmentStatus,
     ChatMessageAssessmentType,
 )
-from unique_toolkit.language_model.invocation_stats import LanguageModelInvocationStats
 
 
 class EvaluationMetricName(StrEnum):
@@ -103,7 +102,6 @@ class EvaluationMetricResult(BaseModel):
     user_info: Optional[str] = None
     error: Exception | None = None
     fact_list: list[str] = Field(default_factory=list[str])
-    invocation_stats: list[LanguageModelInvocationStats] = Field(default_factory=list)
 
 
 class EvaluationAssessmentMessage(BaseModel):

{unique_toolkit-2026.30.0 → unique_toolkit-2026.30.0.dev0}/unique_toolkit/agentic/evaluation/tests/test_context_relevancy_service.py

@@ -16,7 +16,6 @@ from unique_toolkit.agentic.evaluation.schemas import (
 from unique_toolkit.language_model.schemas import (
     LanguageModelAssistantMessage,
     LanguageModelCompletionChoice,
-    LanguageModelTokenUsage,
 )
 from unique_toolkit.language_model.service import LanguageModelResponse
 
@@ -272,73 +271,3 @@ async def test_analyze__raises_evaluator_exception__with_unknown_error(
         assert "Unknown error occurred during context relevancy metric analysis" in str(
             exc_info.value
         )
-
-
-@pytest.mark.ai
-@pytest.mark.asyncio
-async def test_analyze__preserves_invocation_stats__on_regular_output_read_error(
-    context_relevancy_evaluator: MagicMock,
-    sample_evaluation_input: EvaluationMetricInput,
-    basic_evaluation_config: EvaluationMetricConfig,
-) -> None:
-    """
-    Purpose: A billed LLM call that returns no choices must still surface its
-    usage, instead of losing it to the generic error path.
-    Why this matters: `result.choices[0]` access is not otherwise guarded; an
-    empty `choices` list must not silently drop invocation_stats.
-    """
-    mock_result: LanguageModelResponse = LanguageModelResponse(
-        choices=[],
-        usage=LanguageModelTokenUsage(
-            completion_tokens=1, prompt_tokens=2, total_tokens=3
-        ),
-    )
-
-    with patch.object(
-        context_relevancy_evaluator.language_model_service,
-        "complete_async",
-        return_value=mock_result,
-    ):
-        with pytest.raises(EvaluatorException) as exc_info:
-            await context_relevancy_evaluator.analyze(
-                sample_evaluation_input, basic_evaluation_config
-            )
-
-    assert len(exc_info.value.invocation_stats) == 1
-    assert exc_info.value.invocation_stats[0].token_usage.total_tokens == 3
-
-
-@pytest.mark.ai
-@pytest.mark.asyncio
-async def test_analyze__preserves_invocation_stats__on_structured_output_read_error(
-    context_relevancy_evaluator: MagicMock,
-    sample_evaluation_input: EvaluationMetricInput,
-    structured_evaluation_config: EvaluationMetricConfig,
-) -> None:
-    """
-    Purpose: Same as the regular-output case, for the structured-output path.
-    """
-    mock_result: LanguageModelResponse = LanguageModelResponse(
-        choices=[],
-        usage=LanguageModelTokenUsage(
-            completion_tokens=1, prompt_tokens=2, total_tokens=3
-        ),
-    )
-    structured_output_schema: type[EvaluationSchemaStructuredOutput] = (
-        EvaluationSchemaStructuredOutput
-    )
-
-    with patch.object(
-        context_relevancy_evaluator.language_model_service,
-        "complete_async",
-        return_value=mock_result,
-    ):
-        with pytest.raises(EvaluatorException) as exc_info:
-            await context_relevancy_evaluator.analyze(
-                sample_evaluation_input,
-                structured_evaluation_config,
-                structured_output_schema,
-            )
-
-    assert len(exc_info.value.invocation_stats) == 1
-    assert exc_info.value.invocation_stats[0].token_usage.total_tokens == 3