unique_orchestrator 2026.30.0.dev4__tar.gz → 2026.30.0.dev5__tar.gz

{unique_orchestrator-2026.30.0.dev4 → unique_orchestrator-2026.30.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-orchestrator
-Version: 2026.30.0.dev4
+Version: 2026.30.0.dev5
 Summary: 
 Author: Andreas Hauri
 Author-email: Andreas Hauri <andreas.hauri@unique.ai>
@@ -9,7 +9,7 @@ Requires-Dist: pydantic>=2.8.2,<3
 Requires-Dist: pydantic-settings>=2.10.1,<3
 Requires-Dist: typing-extensions>=4.9.0,<5
 Requires-Dist: jinja2>=3.1.0,<4
-Requires-Dist: unique-toolkit>=2026.30.0.dev14,<2026.30.0rc0
+Requires-Dist: unique-toolkit>=2026.30.0.dev15,<2026.30.0rc0
 Requires-Dist: unique-stock-ticker>=2026.30.0.dev0,<2026.30.0rc0
 Requires-Dist: unique-follow-up-questions>=2026.30.0.dev0,<2026.30.0rc0
 Requires-Dist: unique-internal-search>=2026.30.0.dev0,<2026.30.0rc0

{unique_orchestrator-2026.30.0.dev4 → unique_orchestrator-2026.30.0.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_orchestrator"
-version = "2026.30.0.dev4"
+version = "2026.30.0.dev5"
 description = ""
 readme = "README.md"
 license = { text = "Proprietary" }
@@ -13,7 +13,7 @@ dependencies = [
     "pydantic-settings>=2.10.1,<3",
     "typing-extensions>=4.9.0,<5",
     "jinja2>=3.1.0,<4",
-    "unique-toolkit>=2026.30.0.dev14,<2026.30.0rc0",
+    "unique-toolkit>=2026.30.0.dev15,<2026.30.0rc0",
     "unique-stock-ticker>=2026.30.0.dev0,<2026.30.0rc0",
     "unique-follow-up-questions>=2026.30.0.dev0,<2026.30.0rc0",
     "unique-internal-search>=2026.30.0.dev0,<2026.30.0rc0",

{unique_orchestrator-2026.30.0.dev4 → unique_orchestrator-2026.30.0.dev5}/unique_orchestrator/tests/test_unique_ai_execution_timing.py

@@ -1,4 +1,5 @@
 import time
+from datetime import datetime, timezone
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -735,6 +736,27 @@ class TestRunExecutionTimingIntegration:
         assert "loop_iterations" in payload
         assert "total_time" in payload
 
+    @pytest.mark.ai
+    @pytest.mark.asyncio
+    async def test_run__analytics__includes_total_time_to_answer_ms(
+        self, monkeypatch
+    ) -> None:
+        """
+        Purpose: Verify analytics measures from user creation through completion.
+        Why this matters: Internal loop timing excludes queueing and completion latency.
+        Setup summary: Return a known completion timestamp and assert the full delta.
+        """
+        ua = self._build_run_ua(monkeypatch)
+        ua._event.payload.user_message.created_at = "2026-07-13T20:00:00Z"
+        ua._chat_service.modify_assistant_message_async.return_value = MagicMock(
+            completed_at=datetime(2026, 7, 13, 20, 0, 2, 345000, tzinfo=timezone.utc)
+        )
+
+        await ua.run()
+
+        analytics_call = ua._debug_info_manager.add_analytics.call_args
+        assert analytics_call.kwargs["total_time_to_answer_ms"] == 2345
+
     @pytest.mark.ai
     @pytest.mark.asyncio
     async def test_run__execution_times__contains_one_entry_after_single_iteration(
@@ -836,10 +858,10 @@ class TestRunExecutionTimingIntegration:
     ) -> None:
         """
         Purpose: Verify that a failure in update_debug_info_async propagates to the caller.
-        Why this matters: After refactoring, debug info failures are no longer silently
-        swallowed; callers must handle them.
+        Why this matters: Debug failures remain visible without preventing answer
+        completion.
         Setup summary: Make update_debug_info_async raise; run; assert RuntimeError
-        propagates and modify_assistant_message_async is never reached.
+        propagates after modify_assistant_message_async completes the answer.
         """
         ua = self._build_run_ua(monkeypatch)
         ua._chat_service.update_debug_info_async.side_effect = RuntimeError("boom")
@@ -847,7 +869,9 @@ class TestRunExecutionTimingIntegration:
         with pytest.raises(RuntimeError, match="boom"):
             await ua.run()
 
-        ua._chat_service.modify_assistant_message_async.assert_not_called()
+        ua._chat_service.modify_assistant_message_async.assert_awaited_once_with(
+            set_completed_at=True
+        )
 
     @pytest.mark.ai
     @pytest.mark.asyncio

{unique_orchestrator-2026.30.0.dev4 → unique_orchestrator-2026.30.0.dev5}/unique_orchestrator/tests/test_unique_ai_loop_debug_params.py

@@ -76,6 +76,7 @@ def _make_loop_response() -> MagicMock:
     response.message = MagicMock()
     response.message.references = []
     response.message.text = "response text"
+    response.message.original_text = "original response text"
     response.is_empty.return_value = False
     return response
 
@@ -256,6 +257,9 @@ class TestRunLoopDebugParams:
         mock_chat_service.cancellation = mock_cancellation
         mock_chat_service.get_debug_info_async = AsyncMock(return_value={})
         mock_chat_service.update_debug_info_async = AsyncMock(return_value=None)
+        # Returns None by default → no completed_at, so total_time_to_answer_ms
+        # resolves to None. Tests that exercise timing override this return value
+        # with a MagicMock(completed_at=...).
         mock_chat_service.modify_assistant_message_async = AsyncMock(return_value=None)
         mock_chat_service.create_assistant_message_async = AsyncMock(
             return_value=MagicMock(id="assist_new")
@@ -279,6 +283,9 @@ class TestRunLoopDebugParams:
         mock_config = MagicMock()
         mock_config.effective_max_loop_iterations = 1
         mock_config.agent.prompt_config.user_metadata = []
+        mock_config.space.language_model.name = "AZURE_GPT_5_2025_0807"
+        mock_config.space.language_model.family = "openai"
+        mock_config.space.language_model.provider = "AZURE"
 
         mock_history_manager = MagicMock()
         mock_history_manager.get_history_for_model_call = AsyncMock(
@@ -345,6 +352,10 @@ class TestRunLoopDebugParams:
         as the first argument.
         """
         ua = self._build_run_ua(monkeypatch)
+        tool = MagicMock()
+        tool.name = "InternalSearch"
+        tool.display_name.return_value = "Knowledge Base Search"
+        ua._tool_manager.available_tools = [tool]
 
         await ua.run()
 
@@ -353,14 +364,119 @@ class TestRunLoopDebugParams:
         ]
         assert "loop_params" in keys_written
         assert "skills" in keys_written
+        # This test owns one contract: run() forwards the same skills value it
+        # wrote under "skills" as the first positional arg to add_analytics.
+        # Per-field kwargs (references / timing / lengths) are covered by the
+        # focused sibling tests below, so we don't re-pin the whole call here.
         skills_call = next(
             c
             for c in ua._debug_info_manager.add.call_args_list
             if c.args[0] == "skills"
         )
-        ua._debug_info_manager.add_analytics.assert_called_once_with(
-            skills_call.args[1]
-        )
+        ua._debug_info_manager.add_analytics.assert_called_once()
+        analytics_call = ua._debug_info_manager.add_analytics.call_args
+        assert analytics_call.args[0] == skills_call.args[1]
+        assert analytics_call.kwargs["tool_display_names"] == {
+            "InternalSearch": "Knowledge Base Search"
+        }
+
+    @pytest.mark.ai
+    @pytest.mark.asyncio
+    async def test_run__analytics__counts_unique_reference_sources(
+        self, monkeypatch
+    ) -> None:
+        ua = self._build_run_ua(monkeypatch)
+        references = [
+            MagicMock(url="source_1"),
+            MagicMock(url="source_1"),
+            MagicMock(url="source_2"),
+            MagicMock(url="source_3"),
+            MagicMock(url="source_3"),
+        ]
+        ua._loop_iteration_runner.return_value.message.references = references
+        ua._reference_manager.get_references.return_value = [references]
+
+        await ua.run()
+
+        analytics_call = ua._debug_info_manager.add_analytics.call_args
+        assert analytics_call.kwargs["references"] == 3
+
+    @pytest.mark.ai
+    @pytest.mark.asyncio
+    async def test_run__analytics__counts_references_across_iterations(
+        self, monkeypatch
+    ) -> None:
+        """
+        Purpose: Verify analytics includes reference sources from every loop iteration.
+        Why this matters: Earlier citations belong to the same run and must not be
+        omitted when a later iteration appends another reference batch.
+        Setup summary: Return two stored batches with one duplicate source and assert
+        that all three distinct sources are counted.
+        """
+        ua = self._build_run_ua(monkeypatch)
+        first_iteration_references = [
+            MagicMock(url="source_1"),
+            MagicMock(url="source_2"),
+        ]
+        latest_iteration_references = [
+            MagicMock(url="source_2"),
+            MagicMock(url="source_3"),
+        ]
+        ua._reference_manager.get_references.return_value = [
+            first_iteration_references,
+            latest_iteration_references,
+        ]
+
+        await ua.run()
+
+        analytics_call = ua._debug_info_manager.add_analytics.call_args
+        assert analytics_call.kwargs["references"] == 3
+
+    @pytest.mark.ai
+    @pytest.mark.asyncio
+    async def test_run__analytics__counts_distinct_references_without_urls(
+        self, monkeypatch
+    ) -> None:
+        """
+        Purpose: Verify URL-less references are identified by source and source ID.
+        Why this matters: MCP citations can omit URLs and must not collapse into one
+        analytics source.
+        Setup summary: Run with two distinct URL-less references and assert both count.
+        """
+        ua = self._build_run_ua(monkeypatch)
+        references = [
+            MagicMock(url=None, source="mcp", source_id="resource_1"),
+            MagicMock(url=None, source="mcp", source_id="resource_2"),
+        ]
+        ua._loop_iteration_runner.return_value.message.references = references
+        ua._reference_manager.get_references.return_value = [references]
+
+        await ua.run()
+
+        analytics_call = ua._debug_info_manager.add_analytics.call_args
+        assert analytics_call.kwargs["references"] == 2
+
+    @pytest.mark.ai
+    @pytest.mark.asyncio
+    async def test_run__analytics__total_time_to_answer_ms_none_without_completed_at(
+        self, monkeypatch
+    ) -> None:
+        """
+        Purpose: Verify total_time_to_answer_ms is None when the completed message
+        carries no completed_at.
+        Why this matters: Makes the None case deliberate — run() must degrade to
+        None (not crash) when completion time is unavailable, e.g. a tool took
+        control. Pairs with test_run__analytics__includes_total_time_to_answer_ms
+        (execution-timing suite) which covers the populated case.
+        Setup summary: modify_assistant_message_async returns None (fixture default);
+        assert the kwarg is present and None.
+        """
+        ua = self._build_run_ua(monkeypatch)
+
+        await ua.run()
+
+        analytics_call = ua._debug_info_manager.add_analytics.call_args
+        assert analytics_call.kwargs["total_time_to_answer_ms"] is None
 
     @pytest.mark.ai
     @pytest.mark.asyncio

{unique_orchestrator-2026.30.0.dev4 → unique_orchestrator-2026.30.0.dev5}/unique_orchestrator/unique_ai.py

@@ -1,6 +1,6 @@
 import asyncio
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from logging import Logger
 from typing import Any, cast, overload
 
@@ -8,6 +8,7 @@ import jinja2
 from typing_extensions import deprecated
 from unique_skill_tool.service import SkillTool
 from unique_toolkit.agentic.debug_info_manager.debug_info_manager import (
+    AnalyticsLanguageModel,
     DebugInfoManager,
 )
 from unique_toolkit.agentic.evaluation.evaluation_manager import EvaluationManager
@@ -261,9 +262,8 @@ class UniqueAI:
                     self._finalize_loop_timing(loop_start)
                     break
 
-                self._reference_manager.add_references(
-                    loop_response.message.references or []
-                )
+                references = loop_response.message.references or []
+                self._reference_manager.add_references(references)
                 self._last_assistant_text = (
                     loop_response.message.original_text or loop_response.message.text
                 )
@@ -318,12 +318,53 @@ class UniqueAI:
             self._debug_info_manager.add("loop_params", self._loop_debug_params)
             skills_debug_info = self._get_activated_skills_debug_info()
             self._debug_info_manager.add("skills", skills_debug_info)
-            self._debug_info_manager.add_analytics(skills_debug_info)
-
+            reference_sources = {
+                ("url", reference.url)
+                if reference.url is not None
+                else ("source", reference.source, reference.source_id)
+                for references in self._reference_manager.get_references()
+                for reference in references
+            }
+            language_model = self._config.space.language_model
+            tool_display_names = {
+                str(tool.name): tool.display_name() or str(tool.name)
+                for tool in self._tool_manager.available_tools
+            }
             tool_names = [
                 tool["name"] for tool in self._debug_info_manager.get()["tools"]
             ]
 
+            total_time_to_answer_ms: int | None = None
+            if not self._chat_service.cancellation.is_cancelled:
+                if self._config.agent.input_token_distribution.enable_tool_call_persistence:
+                    await self._persist_tool_calls()
+                completed_message = (
+                    await self._chat_service.modify_assistant_message_async(
+                        set_completed_at=not self._tool_took_control,
+                    )
+                )
+                total_time_to_answer_ms = self._calculate_total_time_to_answer_ms(
+                    user_message_created_at=self._event.payload.user_message.created_at,
+                    assistant_completed_at=getattr(
+                        completed_message, "completed_at", None
+                    ),
+                )
+
+            self._debug_info_manager.add_analytics(
+                skills_debug_info,
+                language_model=AnalyticsLanguageModel(
+                    name=str(language_model.name),
+                    family=str(language_model.family),
+                    provider=str(language_model.provider),
+                ),
+                tool_display_names=tool_display_names,
+                references=len(reference_sources),
+                user_prompt_length=len(self._event.payload.user_message.text),
+                answer_length=len(self._last_assistant_text or ""),
+                loop_iteration_count=len(self._execution_times),
+                total_time_to_answer_ms=total_time_to_answer_ms,
+            )
+
             # Get current debug info from chat service and add debug info from run. Do not update if DeepResearch is in the tool names.
             if "DeepResearch" not in tool_names:
                 debug_info = {
@@ -331,16 +372,30 @@ class UniqueAI:
                     **self._debug_info_manager.get(),
                 }
                 await self._chat_service.update_debug_info_async(debug_info=debug_info)
-
-            if not self._chat_service.cancellation.is_cancelled:
-                if self._config.agent.input_token_distribution.enable_tool_call_persistence:
-                    await self._persist_tool_calls()
-                await self._chat_service.modify_assistant_message_async(
-                    set_completed_at=not self._tool_took_control,
-                )
         finally:
             sub.cancel()
 
+    @staticmethod
+    def _calculate_total_time_to_answer_ms(
+        user_message_created_at: str,
+        assistant_completed_at: datetime | None,
+    ) -> int | None:
+        if assistant_completed_at is None:
+            return None
+
+        try:
+            user_created_at = datetime.fromisoformat(user_message_created_at)
+        except (ValueError, TypeError):
+            # ValueError: malformed timestamp; TypeError: created_at not a str.
+            return None
+
+        if user_created_at.tzinfo is None:
+            user_created_at = user_created_at.replace(tzinfo=timezone.utc)
+        if assistant_completed_at.tzinfo is None:
+            assistant_completed_at = assistant_completed_at.replace(tzinfo=timezone.utc)
+
+        return round((assistant_completed_at - user_created_at).total_seconds() * 1000)
+
     def _record_loop_debug_params(self, other_options: dict) -> None:
         reasoning = other_options.get("reasoning")
         thinking_level: str = (