unique_toolkit 2026.28.4__tar.gz → 2026.30.0__tar.gz

{unique_toolkit-2026.28.4 → unique_toolkit-2026.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 2026.28.4
+Version: 2026.30.0
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
 Requires-Dist: openai>=1.105.0,<3
 Requires-Dist: platformdirs>=4.0.0,<5
 Requires-Dist: pillow>=12.1.1
-Requires-Dist: unique-sdk>=2026.28.4
+Requires-Dist: unique-sdk>=2026.30.0
 Requires-Dist: jambo>=0.1.2,<0.2
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: docxtpl>=0.20.1,<0.21
@@ -32,6 +32,7 @@ Requires-Dist: pypandoc>=1.16.2,<2
 Requires-Dist: tenacity>=9.1.4
 Requires-Dist: pydantic-core>=2.41.5
 Requires-Dist: cachetools>=5.0,<6
+Requires-Dist: rapidfuzz>=3.0,<4
 Requires-Dist: fastapi>=0.121.0,<0.122 ; extra == 'fastapi'
 Requires-Dist: starlette>=0.49.1 ; extra == 'fastapi'
 Requires-Dist: uvicorn>=0.37.0,<0.38 ; extra == 'fastapi'

{unique_toolkit-2026.28.4 → unique_toolkit-2026.30.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "2026.28.4"
+version = "2026.30.0"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,7 +28,7 @@ dependencies = [
     "openai>=1.105.0,<3",
     "platformdirs>=4.0.0,<5",
     "pillow>=12.1.1",
-    "unique-sdk>=2026.28.4",
+    "unique-sdk>=2026.30.0",
     "jambo>=0.1.2,<0.2",
     "jinja2>=3.1.6,<4",
     "docxtpl>=0.20.1,<0.21",
@@ -44,6 +44,7 @@ dependencies = [
     "tenacity>=9.1.4",
     "pydantic-core>=2.41.5",
     "cachetools>=5.0,<6",
+    "rapidfuzz>=3.0,<4",
 ]
 
 [project.optional-dependencies]

unique_toolkit-2026.30.0/unique_toolkit/agentic/debug_info_manager/debug_info_manager.py

@@ -0,0 +1,245 @@
+from typing import Any, Required, TypedDict
+
+from unique_toolkit.agentic.tools.openai_builtin import OpenAICodeInterpreterTool
+from unique_toolkit.agentic.tools.openai_builtin.base import OpenAIBuiltInToolName
+from unique_toolkit.agentic.tools.openai_builtin.code_interpreter.postprocessors.generated_files import (
+    ArtifactsDebugInfo,
+)
+from unique_toolkit.agentic.tools.schemas import ToolCallResponse
+from unique_toolkit.agentic.tools.tool_manager import ToolManager
+from unique_toolkit.language_model.schemas import (
+    LanguageModelStreamResponse,
+    ResponsesLanguageModelStreamResponse,
+)
+
+
+class AnalyticsTool(TypedDict, total=False):
+    name: Required[str]
+    display_name: Required[str]
+    is_forced: bool
+    is_exclusive: bool
+    is_sub_agent: bool
+    is_mcp: bool
+    loop_iteration: int
+
+
+class AnalyticsSkill(TypedDict):
+    name: str
+    content_id: str
+    is_forced: bool
+
+
+class AnalyticsLanguageModel(TypedDict):
+    name: str
+    family: str
+    provider: str
+
+
+class AnalyticsToolName(TypedDict):
+    name: str
+    display_name: str
+
+
+class Analytics(TypedDict):
+    answer_length: int
+    artifacts_created_count: int | None
+    artifacts_created_filetype: list[str] | None
+    context_memory_updated: bool | None
+    language_model: AnalyticsLanguageModel
+    loop_iteration_count: int
+    mcp_tool_names_used: list[AnalyticsToolName]
+    # Total size of artifacts/files created this turn, in MiB. None when Code
+    # Interpreter did not run; 0.0 when it ran but produced nothing.
+    output_size: float | None
+    references_count: int
+    skills_used: list[AnalyticsSkill]
+    subagent_names_used: list[AnalyticsToolName]
+    tool_call_count: int
+    tools_used: list[AnalyticsTool]
+    total_time_to_answer_ms: int | None
+    user_prompt_length: int
+
+
+class DebugInfoManager:
+    def __init__(self):
+        self.debug_info: dict[str, Any] = {"tools": []}
+
+    def extract_tool_debug_info(
+        self,
+        tool_call_responses: list[ToolCallResponse],
+        loop_iteration_index: int | None = None,
+    ) -> None:
+        for tool_call_response in tool_call_responses:
+            debug_info = (
+                tool_call_response.debug_info.copy()
+                if tool_call_response.debug_info
+                else {}
+            )
+            tool_info: dict[str, Any] = {
+                "name": tool_call_response.name,
+                "info": debug_info,
+            }
+            if debug_info.get("mcp_server"):
+                tool_info["mcp_server"] = debug_info["mcp_server"]
+            if loop_iteration_index is not None:
+                tool_info["info"]["loop_iteration"] = loop_iteration_index
+            self.debug_info["tools"].append(tool_info)
+
+    def extract_builtin_tool_debug_info(
+        self,
+        stream_response: LanguageModelStreamResponse,
+        tool_manager: ToolManager,
+        loop_iteration_index: int | None = None,
+    ) -> None:
+        self.debug_info["tools"].extend(
+            _extract_tool_calls_from_stream_response(
+                stream_response, tool_manager, loop_iteration_index
+            )
+        )
+
+    def add(self, key: str, value: Any) -> None:
+        self.debug_info = self.debug_info | {key: value}
+
+    def get(self) -> dict[str, Any]:
+        return self.debug_info
+
+    def add_analytics(
+        self,
+        skills: list[dict[str, Any]],
+        language_model: AnalyticsLanguageModel,
+        tool_display_names: dict[str, str],
+        references: int = 0,
+        user_prompt_length: int = 0,
+        answer_length: int = 0,
+        loop_iteration_count: int = 0,
+        total_time_to_answer_ms: int | None = None,
+        artifacts: ArtifactsDebugInfo | None = None,
+        context_memory_updated: bool | None = None,
+    ) -> None:
+        """Add a stable 'analytics' snapshot for downstream ROI/usage reporting.
+
+        Copies the current `tools` list (with each tool's info reduced to
+        attribution fields only), the given `skills` list, usage counts, prompt
+        and answer lengths, and language-model attribution into a new
+        `analytics` key. Later appends to `debug_info["tools"]` (e.g. from
+        extract_tool_debug_info) can't leak into the frozen snapshot. Existing
+        top-level keys (`tools`, `skills`, ...) are left untouched, so this is
+        additive for consumers reading the old shape.
+        """
+        analytics_tools = [
+            _to_analytics_tool_entry(tool, tool_display_names)
+            for tool in self.debug_info.get("tools", [])
+        ]
+        # Artifacts are recorded at the postprocessor seam (Code Interpreter's
+        # DisplayCodeInterpreterFilesPostProcessor) into debug_info["artifacts"].
+        # Absent when no Code Interpreter ran this turn → artifact fields stay
+        # None, preserving the always-present, null-when-unknown contract.
+        analytics = Analytics(
+            tools_used=analytics_tools,
+            tool_call_count=len(analytics_tools),
+            skills_used=[
+                AnalyticsSkill(
+                    name=skill["name"],
+                    content_id=skill["content_id"],
+                    is_forced=skill["is_forced"],
+                )
+                for skill in skills
+            ],
+            references_count=references,
+            user_prompt_length=user_prompt_length,
+            answer_length=answer_length,
+            language_model=language_model.copy(),
+            loop_iteration_count=loop_iteration_count,
+            subagent_names_used=_unique_tool_names(analytics_tools, "is_sub_agent"),
+            mcp_tool_names_used=_unique_tool_names(analytics_tools, "is_mcp"),
+            total_time_to_answer_ms=total_time_to_answer_ms,
+            artifacts_created_count=artifacts["count"] if artifacts else None,
+            artifacts_created_filetype=artifacts["filetypes"] if artifacts else None,
+            output_size=artifacts["output_size"] if artifacts else None,
+            # None when the user-memory postprocessor is not activated for this
+            # turn; True/False when it ran and did/didn't update the profile.
+            context_memory_updated=context_memory_updated,
+        )
+        self.add("analytics", analytics)
+
+
+def _unique_tool_names(
+    analytics_tools: list[AnalyticsTool], flag: str
+) -> list[AnalyticsToolName]:
+    """Dedupe the tools carrying `flag` into `{name, display_name}` entries.
+
+    Keyed on the stable `name` so downstream reporting can join on a stable
+    identifier while still carrying the user-facing `display_name` for labels.
+    """
+    unique: dict[str, AnalyticsToolName] = {}
+    for tool in analytics_tools:
+        if tool.get(flag) and tool["name"] not in unique:
+            unique[tool["name"]] = AnalyticsToolName(
+                name=tool["name"],
+                display_name=tool["display_name"],
+            )
+    return list(unique.values())
+
+
+def _to_analytics_tool_entry(
+    tool: dict[str, Any], tool_display_names: dict[str, str]
+) -> AnalyticsTool:
+    """Return an analytics-safe copy of a tool debug-info entry.
+
+    Keeps only tool attribution fields.
+    """
+    info = tool.get("info") or {}
+    name = tool["name"]
+    analytics_tool = AnalyticsTool(
+        name=name,
+        display_name=tool_display_names.get(name) or name,
+    )
+    if "is_forced" in info:
+        analytics_tool["is_forced"] = info["is_forced"]
+    if "is_exclusive" in info:
+        analytics_tool["is_exclusive"] = info["is_exclusive"]
+    if info.get("is_sub_agent"):
+        analytics_tool["is_sub_agent"] = True
+    if info.get("is_mcp") or tool.get("mcp_server"):
+        analytics_tool["is_mcp"] = True
+    if "loop_iteration" in info:
+        analytics_tool["loop_iteration"] = info["loop_iteration"]
+    return analytics_tool
+
+
+def _extract_tool_calls_from_stream_response(
+    stream_response: LanguageModelStreamResponse,
+    tool_manager: ToolManager,
+    loop_iteration_index: int | None = None,
+) -> list[dict[str, Any]]:
+    if not isinstance(stream_response, ResponsesLanguageModelStreamResponse):
+        return []
+
+    seen = set()
+    tool_infos = []
+
+    for code_interpreter_call in stream_response.code_interpreter_calls:
+        if code_interpreter_call.id in seen:
+            continue
+
+        seen.add(code_interpreter_call.id)
+        tool_name = OpenAIBuiltInToolName.CODE_INTERPRETER
+
+        is_exclusive = tool_name in tool_manager.get_exclusive_tools()
+        is_forced = tool_name in tool_manager.get_tool_choices()
+
+        debug_info = OpenAICodeInterpreterTool.get_debug_info(code_interpreter_call)
+
+        if loop_iteration_index is not None:
+            debug_info["loop_iteration"] = loop_iteration_index
+        debug_info["is_exclusive"] = is_exclusive
+        debug_info["is_forced"] = is_forced
+
+        tool_infos.append(
+            {
+                "name": tool_name,
+                "info": debug_info,
+            }
+        )
+
+    return tool_infos

{unique_toolkit-2026.28.4 → unique_toolkit-2026.30.0}/unique_toolkit/agentic/evaluation/context_relevancy/service.py

@@ -32,6 +32,9 @@ from unique_toolkit.language_model.infos import (
     LanguageModelInfo,
     ModelCapabilities,
 )
+from unique_toolkit.language_model.invocation_stats import (
+    LanguageModelInvocationStats,
+)
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessages,
     LanguageModelSystemMessage,
@@ -145,6 +148,10 @@ class ContextRelevancyEvaluator:
             # Handle regular output
             return await self._handle_regular_output(input, config)
 
+        except EvaluatorException:
+            # Already carries whatever invocation_stats were captured before
+            # the failure -- don't re-wrap it and lose them.
+            raise
         except Exception as e:
             error_message = (
                 "Unknown error occurred during context relevancy metric analysis"
@@ -172,21 +179,49 @@ class ContextRelevancyEvaluator:
             other_options=config.additional_llm_options,
         )
 
+        # Captured as soon as the (billable) LLM call returns, so a validation
+        # failure below can still report it instead of losing it.
+        invocation_stats = (
+            [
+                LanguageModelInvocationStats.from_usage(
+                    config.language_model.name,
+                    result.usage,
+                    source="context_relevancy",
+                )
+            ]
+            if result.usage is not None
+            else []
+        )
+
         try:
             result_content = EvaluationSchemaStructuredOutput.model_validate(
                 result.choices[0].message.parsed
             )
+            evaluation_result = parse_eval_metric_result_structured_output(
+                result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+            )
         except ValidationError as e:
             error_message = "Error occurred during structured output validation of the context relevancy evaluation."
             raise EvaluatorException(
                 error_message=error_message,
                 user_message=error_message,
                 exception=e,
+                invocation_stats=invocation_stats,
+            )
+        except Exception as e:
+            # Catches anything else reading the response (e.g. an empty
+            # `choices` list) -- must still carry invocation_stats, not just
+            # the validation-specific failure above.
+            error_message = "Error occurred while reading the structured output response for context relevancy evaluation."
+            raise EvaluatorException(
+                error_message=f"{error_message}: {e}",
+                user_message=error_message,
+                exception=e,
+                invocation_stats=invocation_stats,
             )
 
-        return parse_eval_metric_result_structured_output(
-            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
-        )
+        evaluation_result.invocation_stats = invocation_stats
+        return evaluation_result
 
     async def _handle_regular_output(
         self,
@@ -201,17 +236,52 @@ class ContextRelevancyEvaluator:
             other_options=config.additional_llm_options,
         )
 
-        result_content = result.choices[0].message.content
-        if not result_content or not isinstance(result_content, str):
-            error_message = "Context relevancy evaluation did not return a result."
+        # Captured as soon as the (billable) LLM call returns, so a failure
+        # below can still report it instead of losing it.
+        invocation_stats = (
+            [
+                LanguageModelInvocationStats.from_usage(
+                    config.language_model.name,
+                    result.usage,
+                    source="context_relevancy",
+                )
+            ]
+            if result.usage is not None
+            else []
+        )
+
+        try:
+            result_content = result.choices[0].message.content
+            if not result_content or not isinstance(result_content, str):
+                error_message = "Context relevancy evaluation did not return a result."
+                raise EvaluatorException(
+                    error_message=error_message,
+                    user_message=error_message,
+                    invocation_stats=invocation_stats,
+                )
+            evaluation_result = parse_eval_metric_result(
+                result_content, EvaluationMetricName.CONTEXT_RELEVANCY
+            )
+        except EvaluatorException as e:
+            # Covers both the raise above and parse_eval_metric_result's own
+            # EvaluatorException (which has no invocation_stats, since it has
+            # no usage context) -- attach ours if it didn't already carry
+            # stats, instead of letting it propagate stats-less.
+            if not e.invocation_stats:
+                e.invocation_stats = invocation_stats
+            raise
+        except Exception as e:
+            # Catches anything else reading the response (e.g. an empty
+            # `choices` list).
+            error_message = "Error occurred while reading the context relevancy evaluation response."
             raise EvaluatorException(
-                error_message=error_message,
+                error_message=f"{error_message}: {e}",
                 user_message=error_message,
+                exception=e,
+                invocation_stats=invocation_stats,
             )
-
-        return parse_eval_metric_result(
-            result_content, EvaluationMetricName.CONTEXT_RELEVANCY
-        )
+        evaluation_result.invocation_stats = invocation_stats
+        return evaluation_result
 
     def _compose_msgs(
         self,

{unique_toolkit-2026.28.4 → unique_toolkit-2026.30.0}/unique_toolkit/agentic/evaluation/evaluation_manager.py

@@ -7,6 +7,7 @@ from unique_toolkit._common.execution import (
     Result,
     SafeTaskExecutor,
 )
+from unique_toolkit.agentic.evaluation.exception import EvaluatorException
 from unique_toolkit.agentic.evaluation.schemas import (
     EvaluationAssessmentMessage,
     EvaluationMetricName,
@@ -17,6 +18,9 @@ from unique_toolkit.chat.schemas import (
     ChatMessageAssessmentType,
 )
 from unique_toolkit.chat.service import ChatService
+from unique_toolkit.language_model.invocation_stats import (
+    LanguageModelInvocationStats,
+)
 from unique_toolkit.language_model.schemas import (
     LanguageModelStreamResponse,
 )
@@ -88,6 +92,7 @@ class EvaluationManager:
         self._evaluations: dict[EvaluationMetricName, Evaluation] = {}
         self._evaluation_passed: bool = True
         self._execution_times: dict[str, float] = {}
+        self._invocation_stats: list[LanguageModelInvocationStats] = []
 
     def add_evaluation(self, evaluation: Evaluation):
         self._evaluations[evaluation.get_name()] = evaluation
@@ -102,6 +107,7 @@ class EvaluationManager:
         assistant_message_id: str,
     ) -> list[EvaluationMetricResult]:
         self._execution_times = {}
+        self._invocation_stats = []
 
         task_executor = SafeTaskExecutor(
             logger=self._logger,
@@ -125,6 +131,7 @@ class EvaluationManager:
             )
             if not unpacked_evaluation_result.is_positive:
                 self._evaluation_passed = False
+            self._invocation_stats.extend(unpacked_evaluation_result.invocation_stats)
             evaluation_results_unpacked.append(unpacked_evaluation_result)
 
         for evaluation_name, evaluation_result in zip(
@@ -141,6 +148,10 @@ class EvaluationManager:
     def get_execution_times(self) -> dict[str, float]:
         return self._execution_times.copy()
 
+    def get_invocation_stats(self) -> list[LanguageModelInvocationStats]:
+        """Per-LLM-call stats from every evaluation that made its own LLM call."""
+        return list(self._invocation_stats)
+
     async def execute_evaluation_call(
         self,
         evaluation_name: EvaluationMetricName,
@@ -178,12 +189,18 @@ class EvaluationManager:
         evaluation_name: EvaluationMetricName,
     ) -> EvaluationMetricResult:
         if not result.success:
+            invocation_stats = (
+                result.exception.invocation_stats
+                if isinstance(result.exception, EvaluatorException)
+                else []
+            )
             return EvaluationMetricResult(
                 name=evaluation_name,
                 is_positive=True,
                 value="RED",
                 reason=str(result.exception),
                 error=Exception("Evaluation result is not successful"),
+                invocation_stats=invocation_stats,
             )
         return result.unpack()
 

unique_toolkit-2026.30.0/unique_toolkit/agentic/evaluation/exception.py

@@ -0,0 +1,23 @@
+from unique_toolkit._common.exception import CommonException
+from unique_toolkit.language_model.invocation_stats import LanguageModelInvocationStats
+
+
+class EvaluatorException(CommonException):
+    def __init__(
+        self,
+        user_message: str,
+        error_message: str,
+        exception: Exception | None = None,
+        invocation_stats: list[LanguageModelInvocationStats] | None = None,
+    ) -> None:
+        super().__init__(
+            user_message=user_message,
+            error_message=error_message,
+            exception=exception,
+        )
+        # Carries whatever usage was already captured before the failure, so a
+        # successful (billable) LLM call isn't silently dropped from the report
+        # just because something after it (parsing, validation) raised.
+        self.invocation_stats: list[LanguageModelInvocationStats] = (
+            invocation_stats or []
+        )

{unique_toolkit-2026.28.4 → unique_toolkit-2026.30.0}/unique_toolkit/agentic/evaluation/hallucination/utils.py

@@ -18,6 +18,9 @@ from unique_toolkit.content import ContentReference
 from unique_toolkit.content.schemas import ContentChunk
 from unique_toolkit.language_model.builder import MessagesBuilder
 from unique_toolkit.language_model.infos import ModelCapabilities
+from unique_toolkit.language_model.invocation_stats import (
+    LanguageModelInvocationStats,
+)
 from unique_toolkit.language_model.schemas import (
     LanguageModelMessages,
     LanguageModelStreamResponse,
@@ -73,29 +76,58 @@ async def check_hallucination(
 
     try:
         msgs = _get_msgs(input, config)
-
         result = await LanguageModelService.complete_async_util(
             company_id=company_id, user_id=user_id, messages=msgs, model_name=model_name
         )
+    except Exception as e:
+        error_message = "Error occurred during hallucination metric analysis"
+        raise EvaluatorException(
+            error_message=f"{error_message}: {e}",
+            user_message=error_message,
+            exception=e,
+        )
+
+    # Captured as soon as the (billable) LLM call returns, so a failure below
+    # (empty content, parsing) can still report it instead of losing it.
+    invocation_stats = (
+        [
+            LanguageModelInvocationStats.from_usage(
+                model_name, result.usage, source="hallucination"
+            )
+        ]
+        if result.usage is not None
+        else []
+    )
+
+    try:
         result_content = result.choices[0].message.content
         if not result_content or not isinstance(result_content, str):
             error_message = "Hallucination evaluation did not return a text result."
             raise EvaluatorException(
                 error_message=error_message,
                 user_message=error_message,
+                invocation_stats=invocation_stats,
             )
-        result = parse_eval_metric_result(
+        eval_result = parse_eval_metric_result(
             result_content,
             EvaluationMetricName.HALLUCINATION,
         )
-
-        return result
+        eval_result.invocation_stats = invocation_stats
+        return eval_result
+    except EvaluatorException as e:
+        # parse_eval_metric_result raises its own EvaluatorException (with no
+        # invocation_stats, since it has no usage context) -- attach ours if
+        # it didn't already carry stats, instead of re-raising it stats-less.
+        if not e.invocation_stats:
+            e.invocation_stats = invocation_stats
+        raise
     except Exception as e:
         error_message = "Error occurred during hallucination metric analysis"
         raise EvaluatorException(
             error_message=f"{error_message}: {e}",
             user_message=error_message,
             exception=e,
+            invocation_stats=invocation_stats,
         )
 
 

{unique_toolkit-2026.28.4 → unique_toolkit-2026.30.0}/unique_toolkit/agentic/evaluation/schemas.py

@@ -10,6 +10,7 @@ from unique_toolkit.chat.schemas import (
     ChatMessageAssessmentStatus,
     ChatMessageAssessmentType,
 )
+from unique_toolkit.language_model.invocation_stats import LanguageModelInvocationStats
 
 
 class EvaluationMetricName(StrEnum):
@@ -102,6 +103,7 @@ class EvaluationMetricResult(BaseModel):
     user_info: Optional[str] = None
     error: Exception | None = None
     fact_list: list[str] = Field(default_factory=list[str])
+    invocation_stats: list[LanguageModelInvocationStats] = Field(default_factory=list)
 
 
 class EvaluationAssessmentMessage(BaseModel):