unique_toolkit 1.63.2__tar.gz → 1.64.1__tar.gz

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 1.63.2
+Version: 1.64.1
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "1.63.2"
+version = "1.64.1"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/feature_flags/feature_flags.py

@@ -82,6 +82,11 @@ class FeatureFlags(BaseSettings):
         description="Emit codeExecution fences in message.text for code interpreter outputs (UN-17972). When disabled, inline file refs are kept as-is. Can be 'true' or comma-separated company IDs.",
     )
 
+    enable_tool_call_persistence_un_15977: FeatureFlag = Field(
+        default=FeatureFlag(False),
+        description="Persist tool calls and reconstruct tool call history across turns (UN-15977). Can be 'true' or comma-separated company IDs.",
+    )
+
     model_config = SettingsConfigDict(
         extra="ignore",
         env_prefix="FEATURE_FLAG_",

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/history_manager/history_manager.py

@@ -77,6 +77,11 @@ class HistoryManagerConfig(BaseModel):
             * self.percent_of_max_tokens_for_history,
         )
 
+    enable_tool_call_persistence: bool = Field(
+        default=False,
+        description="When True, tool calls are persisted to the database and reconstructed from prior turns.",
+    )
+
     uploaded_content_config: (
         Annotated[
             UploadedContentConfig,
@@ -125,6 +130,7 @@ class HistoryManager:
             has_uploaded_content_config=bool(self._config.uploaded_content_config),
             language_model=self._language_model,
             reference_manager=reference_manager,
+            enable_tool_call_persistence=self._config.enable_tool_call_persistence,
         )
         self._reference_manager = reference_manager
         self._tool_call_result_history: list[ToolCallResponse] = []

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/history_manager/loop_token_reducer.py

@@ -10,6 +10,7 @@ from unique_toolkit._common.token.token_counting import (
 from unique_toolkit._common.validators import LMI
 from unique_toolkit.agentic.history_manager.history_construction_with_contents import (
     FileContentSerialization,
+    get_full_history_with_contents,
     get_full_history_with_contents_and_tool_calls,
 )
 from unique_toolkit.agentic.history_manager.utils import serialize_tool_content_json
@@ -54,6 +55,7 @@ class LoopTokenReducer:
         has_uploaded_content_config: bool,
         reference_manager: ReferenceManager,
         language_model: LMI,
+        enable_tool_call_persistence: bool = False,
     ):
         self._max_history_tokens = max_history_tokens
         self._has_uploaded_content_config = has_uploaded_content_config
@@ -73,6 +75,7 @@ class LoopTokenReducer:
         )
         self._max_db_source_number: int = -1
         self._db_source_map: dict[int, ContentChunk] = {}
+        self._enable_tool_call_persistence = enable_tool_call_persistence
 
     @property
     def max_db_source_number(self) -> int:
@@ -303,19 +306,31 @@ class LoopTokenReducer:
         Returns:
             list[LanguageModelMessage]: The history
         """
-        full_history, max_src, src_map = get_full_history_with_contents_and_tool_calls(
-            user_message=self._user_message,
-            chat_id=self._chat_id,
-            chat_service=self._chat_service,
-            content_service=self._content_service,
-            file_content_serialization_type=(
-                FileContentSerialization.NONE
-                if self._has_uploaded_content_config
-                else FileContentSerialization.FILE_NAME
-            ),
+        file_content_serialization_type = (
+            FileContentSerialization.NONE
+            if self._has_uploaded_content_config
+            else FileContentSerialization.FILE_NAME
         )
-        self._max_db_source_number = max_src
-        self._db_source_map = src_map
+        if self._enable_tool_call_persistence:
+            full_history, max_src, src_map = (
+                get_full_history_with_contents_and_tool_calls(
+                    user_message=self._user_message,
+                    chat_id=self._chat_id,
+                    chat_service=self._chat_service,
+                    content_service=self._content_service,
+                    file_content_serialization_type=file_content_serialization_type,
+                )
+            )
+            self._max_db_source_number = max_src
+            self._db_source_map = src_map
+        else:
+            full_history = get_full_history_with_contents(
+                user_message=self._user_message,
+                chat_id=self._chat_id,
+                chat_service=self._chat_service,
+                content_service=self._content_service,
+                file_content_serialization_type=file_content_serialization_type,
+            )
 
         if remove_from_text is not None:
             full_history.root = await self._clean_messages(

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/tools/openai_builtin/base.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from enum import StrEnum
 from typing import Generic, TypeVar
 
+from openai.types.responses import ResponseIncludable
 from openai.types.responses.tool_param import CodeInterpreter
 
 from unique_toolkit.agentic.tools.schemas import ToolPrompts
@@ -44,3 +45,12 @@ class OpenAIBuiltInTool(ABC, Generic[ToolType]):
     @abstractmethod
     def display_name(self) -> str:
         raise NotImplementedError()
+
+    def get_required_include_params(self) -> list[ResponseIncludable]:
+        """Return Responses API `include` values required by this tool.
+
+        Subclasses override this when they need additional data attached to the
+        response (e.g. code interpreter execution logs).  The default is an
+        empty list, meaning no extra includes are needed.
+        """
+        return []

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/tools/openai_builtin/code_interpreter/postprocessors/code_display.py

@@ -6,6 +6,7 @@ from typing import override
 from pydantic import BaseModel, Field
 from pydantic.json_schema import SkipJsonSchema
 
+from unique_toolkit.agentic.feature_flags.feature_flags import feature_flags
 from unique_toolkit.agentic.postprocessor.postprocessor_manager import (
     ResponsesApiPostprocessor,
 )
@@ -24,22 +25,6 @@ _TEMPLATE = """
 
 """.lstrip()
 
-# Pattern matching the block emitted by _TEMPLATE (including trailing </br>).
-# Used when another postprocessor supersedes this content (e.g. fence injection).
-_EXECUTED_CODE_BLOCK_RE = re.compile(
-    r"\n*<details><summary>Code Interpreter Call</summary>.*?</details>[ \t]*\n*(?:[ \t]*</br>[ \t]*\n*)?",
-    re.DOTALL,
-)
-
-
-def strip_executed_code_blocks(text: str) -> str:
-    """Remove all executed-code <details> blocks emitted by this postprocessor.
-
-    Call this when the same content is represented elsewhere (e.g. imgWithSource
-    / fileWithSource fences) so the message does not contain duplicate code blocks.
-    """
-    return _EXECUTED_CODE_BLOCK_RE.sub("", text)
-
 
 logger = logging.getLogger(__name__)
 
@@ -59,18 +44,32 @@ class ShowExecutedCodePostprocessorConfig(BaseModel):
 
 
 class ShowExecutedCodePostprocessor(ResponsesApiPostprocessor):
-    def __init__(self, config: ShowExecutedCodePostprocessorConfig):
+    def __init__(
+        self,
+        config: ShowExecutedCodePostprocessorConfig,
+        company_id: str | None = None,
+    ):
         super().__init__(self.__class__.__name__)
         self._config = config
+        self._company_id = company_id
 
     @override
     async def run(self, loop_response: ResponsesLanguageModelStreamResponse) -> None:
+        if feature_flags.enable_code_execution_fence_un_17972.is_enabled(
+            self._company_id
+        ):
+            return
         await asyncio.sleep(self._config.sleep_time_before_display)
 
     @override
     def apply_postprocessing_to_response(
         self, loop_response: ResponsesLanguageModelStreamResponse
     ) -> bool:
+        if feature_flags.enable_code_execution_fence_un_17972.is_enabled(
+            self._company_id
+        ):
+            return False
+
         prepended_text = ""
         for output in loop_response.code_interpreter_calls:
             prepended_text += _TEMPLATE.format(code=output.code)
@@ -83,6 +82,5 @@ class ShowExecutedCodePostprocessor(ResponsesApiPostprocessor):
     async def remove_from_text(self, text) -> str:
         if not self._config.remove_from_history:
             return text
-        # Remove code interpreter blocks using regex
         pattern = r"<details><summary>Code Interpreter Call</summary>.*?</details>"
         return re.sub(pattern, "", text, flags=re.DOTALL)

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/tools/openai_builtin/code_interpreter/postprocessors/generated_files.py

@@ -6,6 +6,7 @@ from mimetypes import guess_type
 from typing import NamedTuple, override
 
 from openai import AsyncOpenAI
+from openai.types.responses import ResponseCodeInterpreterToolCall
 from openai.types.responses.response_output_text import AnnotationContainerFileCitation
 from pydantic import BaseModel, Field, RootModel
 from pydantic.json_schema import SkipJsonSchema
@@ -20,9 +21,6 @@ from unique_toolkit.agentic.short_term_memory_manager.persistent_short_term_memo
     PersistentShortMemoryManager,
 )
 from unique_toolkit.agentic.tools.config import get_configuration_dict
-from unique_toolkit.agentic.tools.openai_builtin.code_interpreter.postprocessors.code_display import (
-    strip_executed_code_blocks,
-)
 from unique_toolkit.agentic.tools.openai_builtin.code_interpreter.schemas import (
     CodeInterpreterBlock,
     CodeInterpreterFile,
@@ -110,6 +108,7 @@ class DisplayCodeInterpreterFilesPostProcessor(
         if self._chat_service is None:
             raise ValueError("ChatService is required if uploadToChat is True")
 
+        self._orphan_code_blocks: list[CodeInterpreterBlock] = []
         self._short_term_memory_manager = None
         if chat_id is not None and user_id is not None and company_id is not None:
             self._short_term_memory_manager = _init_short_term_memory_manager(
@@ -150,10 +149,23 @@ class DisplayCodeInterpreterFilesPostProcessor(
             ]
         )
 
+        if feature_flags.enable_code_execution_fence_un_17972.is_enabled(
+            self._company_id
+        ):
+            self._orphan_code_blocks = await self._upload_orphan_code_as_txt(
+                loop_response
+            )
+        else:
+            self._orphan_code_blocks = []
+
     @override
     def apply_postprocessing_to_response(
         self, loop_response: ResponsesLanguageModelStreamResponse
     ) -> bool:
+        if loop_response.message.references is None:
+            loop_response.message.references = []
+        if loop_response.message.text is None:
+            loop_response.message.text = ""
         ref_number = _get_next_ref_number(loop_response.message.references)
         changed = False
 
@@ -248,6 +260,27 @@ class DisplayCodeInterpreterFilesPostProcessor(
             )
             changed |= loop_response.message.text != text_before
 
+            if self._orphan_code_blocks:
+                fence_id = _get_next_fence_id(loop_response.message.text)
+                orphan_fences = _build_orphan_fences(self._orphan_code_blocks, fence_id)
+                loop_response.message.text = (
+                    loop_response.message.text.rstrip() + "\n\n" + orphan_fences
+                )
+                ref_number = _get_next_ref_number(loop_response.message.references)
+                for block in self._orphan_code_blocks:
+                    for file in block.files:
+                        loop_response.message.references.append(
+                            ContentReference(
+                                sequence_number=ref_number,
+                                source_id=file.content_id,
+                                source="node-ingestion-chunks",
+                                url=f"unique://content/{file.content_id}",
+                                name=file.filename,
+                            )
+                        )
+                        ref_number += 1
+                changed = True
+
         _warn_missing_content_ids(loop_response.message.text, self._content_map)
         loop_response.message.text, dangling_replaced = _replace_dangling_sandbox_links(
             loop_response.message.text, self._config.file_download_failed_message
@@ -317,6 +350,110 @@ class DisplayCodeInterpreterFilesPostProcessor(
             _DisplayedFilesShortTermMemorySchema(root=content_infos)
         )
 
+    async def _upload_orphan_code_as_txt(
+        self, loop_response: ResponsesLanguageModelStreamResponse
+    ) -> list[CodeInterpreterBlock]:
+        """Upload stdout (or code) from calls that produced no container files as .txt artifacts.
+
+        When code interpreter runs but generates no files or images, the response
+        text has no sandbox links for fence injection to hook onto.  This method
+        converts those "orphan" calls into downloadable .txt files so the fence
+        postprocessor can attach a fileWithSource component and the new Code
+        Execution UI is shown instead of the legacy <details> block.
+
+        Called only when the fence feature flag is enabled.  Skipped entirely if
+        any container files were produced (normal fencing handles those).
+        """
+        if loop_response.container_files:
+            return []
+
+        calls = loop_response.code_interpreter_calls
+        if not calls:
+            return []
+
+        assert self._chat_service is not None  # Checked in __init__
+
+        use_numeric_suffix = len(calls) > 1
+        orphan_blocks: list[CodeInterpreterBlock] = []
+
+        for i, call in enumerate(calls):
+            if not call.code:
+                continue
+
+            # Code interpreter logs are only present on each code_interpreter_call when
+            # the create/retrieve request sets include=["code_interpreter_call.outputs"]
+            # (see OpenAI Responses API `include` and manual examples in
+            # docs/manual_examples_for_docs/code_execution_openai_client.py).  If
+            # `outputs` is omitted, call.outputs is null and we fall back to the
+            # source code as the downloadable txt.
+            txt_content = _collect_stdout(call) or call.code
+            filename = f"code_{i + 1}.txt" if use_numeric_suffix else "code.txt"
+
+            try:
+                content = await self._chat_service.upload_to_chat_from_bytes_async(
+                    content=txt_content.encode("utf-8"),
+                    content_name=filename,
+                    mime_type="text/plain",
+                    skip_ingestion=True,
+                    hide_in_chat=True,
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to upload orphan code block as txt for call %d ('%s'); "
+                    "falling back to legacy code block display.",
+                    i,
+                    filename,
+                )
+                continue
+
+            orphan_file = CodeInterpreterFile(
+                filename=filename,
+                content_id=content.id,
+                type="document",
+            )
+            orphan_blocks.append(
+                CodeInterpreterBlock(code=call.code, files=[orphan_file])
+            )
+            logger.info(
+                "Uploaded orphan code block as '%s' (content_id=%s)",
+                filename,
+                content.id,
+            )
+
+        return orphan_blocks
+
+
+def _collect_stdout(call: ResponseCodeInterpreterToolCall) -> str:
+    """Collect all stdout logs from a code interpreter call's outputs."""
+    if not call.outputs:
+        return ""
+    return "\n".join(output.logs for output in call.outputs if output.type == "logs")
+
+
+def _get_next_fence_id(text: str) -> int:
+    """Return the next available fence id by scanning existing fences in the text."""
+    existing = re.findall(r"````(?:imgWithSource|fileWithSource)\(id='(\d+)'", text)
+    if not existing:
+        return 1
+    return max(int(fid) for fid in existing) + 1
+
+
+def _build_orphan_fences(
+    blocks: list[CodeInterpreterBlock], start_fence_id: int
+) -> str:
+    """Build concatenated fence text for orphan code blocks.
+
+    Orphan blocks have no inline ref in the message text, so their fences are
+    appended directly rather than injected via pattern replacement.
+    """
+    fence_id = start_fence_id
+    parts: list[str] = []
+    for block in blocks:
+        for file in block.files:
+            parts.append(_build_file_fence(file, block.code, fence_id))
+            fence_id += 1
+    return "\n".join(parts)
+
 
 def _get_file_type(filename: str) -> CodeInterpreterFileType:
     mime = guess_type(filename)[0] or ""
@@ -460,8 +597,6 @@ def _inject_code_execution_fences(
 
     Each file gets its own fence placed at the position of its inline ref. Duplicate
     refs for the same file (overwrite case) are removed after the first is replaced.
-    When at least one fence was injected, executed-code <details> blocks are
-    stripped via strip_executed_code_blocks() from the code_display postprocessor.
 
     fence_id is a message-level counter so each fence has a unique id.
     """
@@ -487,7 +622,6 @@ def _inject_code_execution_fences(
             # Remove duplicate refs (overwrite case)
             text = re.sub(pattern, "", text)
     if any_fence_injected:
-        text = strip_executed_code_blocks(text)
         text = _ensure_fences_are_standalone(text)
         text = _CONSECUTIVE_FENCES_RE.sub(r"\1\n", text)
     return text
@@ -694,7 +828,9 @@ def _warn_unmatched_code_blocks(
             )
 
 
-def _get_next_ref_number(references: list[ContentReference]) -> int:
+def _get_next_ref_number(references: list[ContentReference] | None) -> int:
+    if not references:
+        return 1
     max_ref_number = 0
     for ref in references:
         max_ref_number = max(max_ref_number, ref.sequence_number)

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/tools/openai_builtin/code_interpreter/service.py

@@ -2,7 +2,7 @@ import logging
 from typing import Any, override
 
 from openai import AsyncOpenAI, BaseModel, NotFoundError
-from openai.types.responses import ResponseCodeInterpreterToolCall
+from openai.types.responses import ResponseCodeInterpreterToolCall, ResponseIncludable
 from openai.types.responses.tool_param import CodeInterpreter
 
 from unique_toolkit import ContentService, ShortTermMemoryService
@@ -280,6 +280,14 @@ class OpenAICodeInterpreterTool(OpenAIBuiltInTool[CodeInterpreter]):
     def display_name(self) -> str:
         return self.DISPLAY_NAME
 
+    @override
+    def get_required_include_params(self) -> list[ResponseIncludable]:
+        if feature_flags.enable_code_execution_fence_un_17972.is_enabled(
+            self._company_id
+        ):
+            return ["code_interpreter_call.outputs"]
+        return []
+
     @classmethod
     def get_debug_info(cls, call: ResponseCodeInterpreterToolCall) -> dict[str, Any]:
         return {

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/tools/openai_builtin/manager.py

@@ -1,4 +1,5 @@
 from openai import AsyncOpenAI
+from openai.types.responses import ResponseIncludable
 
 from unique_toolkit.agentic.tools.config import ToolBuildConfig
 from unique_toolkit.agentic.tools.openai_builtin.base import (
@@ -79,3 +80,14 @@ class OpenAIBuiltInToolManager:
 
     def get_all_openai_builtin_tools(self) -> list[OpenAIBuiltInTool]:
         return self._builtin_tools.copy()
+
+    def get_required_include_params(self) -> list[ResponseIncludable]:
+        """Aggregate include params required by all active built-in tools."""
+        seen: set[str] = set()
+        result: list[ResponseIncludable] = []
+        for tool in self._builtin_tools:
+            for param in tool.get_required_include_params():
+                if param not in seen:
+                    seen.add(param)
+                    result.append(param)
+        return result

{unique_toolkit-1.63.2 → unique_toolkit-1.64.1}/unique_toolkit/agentic/tools/tool_manager.py

@@ -7,6 +7,7 @@ from openai.types.chat import (
     ChatCompletionNamedToolChoiceParam,
 )
 from openai.types.responses import (
+    ResponseIncludable,
     ToolParam,
     response_create_params,
 )
@@ -490,6 +491,7 @@ class ResponsesApiToolManager(_ToolManager[Literal["responses"]]):
         a2a_manager: A2AManager,
         builtin_tool_manager: OpenAIBuiltInToolManager,
     ) -> None:
+        self._builtin_tool_manager = builtin_tool_manager
         super().__init__(
             logger=logger,
             config=config,
@@ -500,3 +502,7 @@ class ResponsesApiToolManager(_ToolManager[Literal["responses"]]):
             api_mode="responses",
             builtin_tool_manager=builtin_tool_manager,
         )
+
+    def get_required_include_params(self) -> list[ResponseIncludable]:
+        """Return Responses API include params required by all active built-in tools."""
+        return self._builtin_tool_manager.get_required_include_params()