unique_toolkit 1.14.11__py3-none-any.whl → 1.16.0__py3-none-any.whl

unique_toolkit/_common/api_calling/human_verification_manager.py

@@ -19,7 +19,10 @@ from unique_toolkit._common.endpoint_requestor import (
     RequestorType,
     build_requestor,
 )
-from unique_toolkit._common.pydantic_helpers import create_union_model
+from unique_toolkit._common.pydantic_helpers import (
+    create_complement_model,
+    create_union_model,
+)
 from unique_toolkit._common.string_utilities import (
     dict_to_markdown_table,
     extract_dicts_from_string,
@@ -78,24 +81,67 @@ class HumanVerificationManagerForApiCalling(
             ]
         ],
         requestor_type: RequestorType = RequestorType.REQUESTS,
+        environment_payload_params: BaseModel | None = None,
+        modifiable_payload_params_model: type[BaseModel] | None = None,
         **kwargs: dict[str, Any],
     ):
+        """
+        Manages human verification for api calling.
+
+        Args:
+            logger: The logger to use for logging.
+            operation: The operation to use for the api calling.
+            requestor_type: The requestor type to use for the api calling.
+            environment_payload_params: The environment payload params to use for the api calling.
+                If None, the modifiable params model will be the operation payload model.
+                This can be useful for parameters in the payload that should not be modified by the user.
+            modifiable_payload_params_model: The modifiable payload params model to use for the api calling.
+                If None, a complement model will be created using the operation payload model
+                and the environment payload params.
+                If provided, it will be used instead of the complement model.
+                This is necessary if the modifiable params model is required
+                to use custom validators or serializers.
+            **kwargs: Additional keyword arguments to pass to the requestor.
+        """
         self._logger = logger
         self._operation = operation
-
+        self._environment_payload_params = environment_payload_params
         # Create internal models for this manager instance
 
-        class ConcreteApiCall(BaseModel):
+        if self._environment_payload_params is None:
+            self._modifiable_payload_params_model = self._operation.payload_model()
+        else:
+            if modifiable_payload_params_model is None:
+                self._modifiable_payload_params_model = create_complement_model(
+                    model_type_a=self._operation.payload_model(),
+                    model_type_b=type(self._environment_payload_params),
+                )
+            else:
+                # This is necessary if the modifiable params model is required
+                # to use custom validators or serializers.
+                self._modifiable_payload_params_model = modifiable_payload_params_model
+
+        if self._environment_payload_params is not None:
+            combined_keys = set(
+                self._modifiable_payload_params_model.model_fields.keys()
+            ) | set(type(self._environment_payload_params).model_fields.keys())
+            payload_keys = set(self._operation.payload_model().model_fields.keys())
+            if not payload_keys.issubset(combined_keys):
+                raise ValueError(
+                    "The modifiable params model + the environment parameters do not have all the keys of the operation payload model."
+                )
+
+        class VerificationModel(BaseModel):
             confirmation: HumanConfirmation
-            payload: self._operation.payload_model()  # type: ignore
+            modifiable_params: self._modifiable_payload_params_model  # type: ignore
 
-        self._api_call_model = ConcreteApiCall
+        self._verification_model = VerificationModel
 
+        self._requestor_type = requestor_type
         self._combined_params_model = create_union_model(
             model_type_a=self._operation.path_params_model(),
             model_type_b=self._operation.payload_model(),
         )
-        self._requestor_type = requestor_type
         self._requestor = build_requestor(
             requestor_type=requestor_type,
             operation_type=operation,
@@ -104,7 +150,10 @@ class HumanVerificationManagerForApiCalling(
         )
 
     def detect_api_calls_from_user_message(
-        self, *, last_assistant_message: ChatMessage, user_message: str
+        self,
+        *,
+        last_assistant_message: ChatMessage,
+        user_message: str,
     ) -> PayloadType | None:
         user_message_dicts = extract_dicts_from_string(user_message)
         if len(user_message_dicts) == 0:
@@ -114,13 +163,20 @@ class HumanVerificationManagerForApiCalling(
         for user_message_dict in user_message_dicts:
             try:
                 # Convert dict to payload model first, then create payload
-                api_call = self._api_call_model.model_validate(
+                verfication_data = self._verification_model.model_validate(
                     user_message_dict, by_alias=True, by_name=True
                 )
                 if self._verify_human_verification(
-                    api_call.confirmation, last_assistant_message
+                    verfication_data.confirmation, last_assistant_message
                 ):
-                    return api_call.payload
+                    payload_dict = verfication_data.modifiable_params.model_dump()
+                    if self._environment_payload_params is not None:
+                        payload_dict.update(
+                            self._environment_payload_params.model_dump()
+                        )
+
+                    return self._operation.payload_model().model_validate(payload_dict)
+
             except Exception as e:
                 self._logger.error(f"Error detecting api calls from user message: {e}")
 
@@ -141,11 +197,27 @@ class HumanVerificationManagerForApiCalling(
         return confirmation.payload_hash in last_assistant_message.content
 
     def _create_next_user_message(self, payload: PayloadType) -> str:
-        api_call = self._api_call_model(
-            payload=payload,
+        # Extract only the modifiable fields from the payload
+        payload_dict = payload.model_dump()
+        if self._environment_payload_params is not None:
+            # Remove environment params from payload to avoid validation errors
+            environment_fields = set(
+                type(self._environment_payload_params).model_fields.keys()
+            )
+            modifiable_dict = {
+                k: v for k, v in payload_dict.items() if k not in environment_fields
+            }
+        else:
+            modifiable_dict = payload_dict
+
+        modifiable_params = self._modifiable_payload_params_model.model_validate(
+            modifiable_dict
+        )
+        api_call = self._verification_model(
+            modifiable_params=modifiable_params,
             confirmation=HumanConfirmation(
                 payload_hash=hashlib.sha256(
-                    payload.model_dump_json().encode()
+                    modifiable_params.model_dump_json().encode()
                 ).hexdigest(),
                 time_stamp=datetime.now(),
             ),
@@ -168,6 +240,14 @@ class HumanVerificationManagerForApiCalling(
         path_params: PathParamsType,
         payload: PayloadType,
     ) -> ResponseType:
+        """
+        Call the api with the given path params, payload and secured payload params.
+
+        The `secured payload params` are params that are enforced by the application.
+        It should generally be not possible for the user to adapt those but here we
+        ensure that the application has the last word.
+
+        """
         params = path_params.model_dump()
         params.update(payload.model_dump())
 
@@ -217,8 +297,8 @@ if __name__ == "__main__":
 
     payload = GetUserRequestBody(include_profile=True)
 
-    api_call = human_verification_manager._api_call_model(
-        payload=payload,
+    api_call = human_verification_manager._verification_model(
+        modifiable_params=payload,
         confirmation=HumanConfirmation(
             payload_hash=hashlib.sha256(payload.model_dump_json().encode()).hexdigest(),
             time_stamp=datetime.now(),

unique_toolkit/agentic/postprocessor/postprocessor_manager.py

@@ -6,6 +6,7 @@ from unique_toolkit.agentic.tools.utils.execution.execution import SafeTaskExecu
 from unique_toolkit.chat.service import ChatService
 from unique_toolkit.language_model.schemas import (
     LanguageModelStreamResponse,
+    ResponsesLanguageModelStreamResponse,
 )
 
 
@@ -26,7 +27,30 @@ class Postprocessor(ABC):
             "Subclasses must implement this method to apply post-processing to the response."
         )
 
-    async def remove_from_text(self, text) -> str:
+    async def remove_from_text(self, text: str) -> str:
+        raise NotImplementedError(
+            "Subclasses must implement this method to remove post-processing from the message."
+        )
+
+
+class ResponsesApiPostprocessor(ABC):
+    def __init__(self, name: str):
+        self.name = name
+
+    def get_name(self) -> str:
+        return self.name
+
+    async def run(self, loop_response: ResponsesLanguageModelStreamResponse) -> None:
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def apply_postprocessing_to_response(
+        self, loop_response: ResponsesLanguageModelStreamResponse
+    ) -> bool:
+        raise NotImplementedError(
+            "Subclasses must implement this method to apply post-processing to the response."
+        )
+
+    async def remove_from_text(self, text: str) -> str:
         raise NotImplementedError(
             "Subclasses must implement this method to remove post-processing from the message."
         )
@@ -59,12 +83,16 @@ class PostprocessorManager:
     ):
         self._logger = logger
         self._chat_service = chat_service
-        self._postprocessors: list[Postprocessor] = []
+        self._postprocessors: list[Postprocessor | ResponsesApiPostprocessor] = []
 
-    def add_postprocessor(self, postprocessor: Postprocessor):
+    def add_postprocessor(
+        self, postprocessor: Postprocessor | ResponsesApiPostprocessor
+    ):
         self._postprocessors.append(postprocessor)
 
-    def get_postprocessors(self, name: str) -> list[Postprocessor]:
+    def get_postprocessors(
+        self, name: str
+    ) -> list[Postprocessor | ResponsesApiPostprocessor]:
         return self._postprocessors
 
     async def run_postprocessors(
@@ -75,25 +103,36 @@ class PostprocessorManager:
             logger=self._logger,
         )
 
+        if isinstance(loop_response, ResponsesLanguageModelStreamResponse):
+            postprocessors = self._postprocessors
+        else:
+            postprocessors = [
+                postprocessor
+                for postprocessor in self._postprocessors
+                if isinstance(postprocessor, Postprocessor)
+            ]
+
         tasks = [
             task_executor.execute_async(
                 self.execute_postprocessors,
                 loop_response=loop_response,
                 postprocessor_instance=postprocessor,
             )
-            for postprocessor in self._postprocessors
+            for postprocessor in postprocessors
         ]
         postprocessor_results = await asyncio.gather(*tasks)
 
-        for i, result in enumerate(postprocessor_results):
+        for postprocessor, result in zip(postprocessors, postprocessor_results):
             if not result.success:
                 self._logger.warning(
-                    f"Postprocessor {self._postprocessors[i].get_name()} failed to run."
+                    "Postprocessor %s failed to run.",
+                    postprocessor.get_name(),
+                    exc_info=result.exception,
                 )
 
         modification_results = [
-            postprocessor.apply_postprocessing_to_response(loop_response)
-            for postprocessor in self._postprocessors
+            postprocessor.apply_postprocessing_to_response(loop_response)  # type: ignore
+            for postprocessor in postprocessors
         ]
 
         has_been_modified = any(modification_results)
@@ -108,9 +147,9 @@ class PostprocessorManager:
     async def execute_postprocessors(
         self,
         loop_response: LanguageModelStreamResponse,
-        postprocessor_instance: Postprocessor,
+        postprocessor_instance: Postprocessor | ResponsesApiPostprocessor,
     ) -> None:
-        await postprocessor_instance.run(loop_response)
+        await postprocessor_instance.run(loop_response)  # type: ignore
 
     async def remove_from_text(
         self,

unique_toolkit/agentic/responses_api/__init__.py

@@ -0,0 +1,19 @@
+from unique_toolkit.agentic.responses_api.postprocessors.code_display import (
+    ShowExecutedCodePostprocessor,
+    ShowExecutedCodePostprocessorConfig,
+)
+from unique_toolkit.agentic.responses_api.postprocessors.generated_files import (
+    DisplayCodeInterpreterFilesPostProcessor,
+    DisplayCodeInterpreterFilesPostProcessorConfig,
+)
+from unique_toolkit.agentic.responses_api.stream_handler import (
+    ResponsesStreamingHandler,
+)
+
+__all__ = [
+    "ShowExecutedCodePostprocessor",
+    "ShowExecutedCodePostprocessorConfig",
+    "DisplayCodeInterpreterFilesPostProcessorConfig",
+    "DisplayCodeInterpreterFilesPostProcessor",
+    "ResponsesStreamingHandler",
+]

unique_toolkit/agentic/responses_api/postprocessors/code_display.py

@@ -0,0 +1,63 @@
+import logging
+import re
+from typing import override
+
+from pydantic import BaseModel, Field
+
+from unique_toolkit.agentic.postprocessor.postprocessor_manager import (
+    ResponsesApiPostprocessor,
+)
+from unique_toolkit.agentic.tools.config import get_configuration_dict
+from unique_toolkit.language_model.schemas import ResponsesLanguageModelStreamResponse
+
+_TEMPLATE = """
+<details><summary>Code Interpreter Call</summary>    
+
+```python
+{code}
+```  
+
+</details>    
+</br>
+
+""".lstrip()
+
+logger = logging.getLogger(__name__)
+
+
+class ShowExecutedCodePostprocessorConfig(BaseModel):
+    model_config = get_configuration_dict()
+    remove_from_history: bool = Field(
+        default=False,
+        description="If set, the code interpreter call will be removed from the history on subsequent calls to the assistant.",
+    )
+
+
+class ShowExecutedCodePostprocessor(ResponsesApiPostprocessor):
+    def __init__(self, config: ShowExecutedCodePostprocessorConfig):
+        super().__init__(self.__class__.__name__)
+        self._config = config
+
+    @override
+    async def run(self, loop_response: ResponsesLanguageModelStreamResponse) -> None:
+        return None
+
+    @override
+    def apply_postprocessing_to_response(
+        self, loop_response: ResponsesLanguageModelStreamResponse
+    ) -> bool:
+        prepended_text = ""
+        for output in loop_response.code_interpreter_calls:
+            prepended_text += _TEMPLATE.format(code=output.code)
+
+        loop_response.message.text = prepended_text + loop_response.message.text
+
+        return prepended_text != ""
+
+    @override
+    async def remove_from_text(self, text) -> str:
+        if not self._config.remove_from_history:
+            return text
+        # Remove code interpreter blocks using regex
+        pattern = r"<details><summary>Code Interpreter Call</summary>.*?</details>"
+        return re.sub(pattern, "", text, flags=re.DOTALL)

unique_toolkit/agentic/responses_api/postprocessors/generated_files.py

@@ -0,0 +1,145 @@
+import logging
+import re
+from mimetypes import guess_type
+from typing import override
+
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+from unique_sdk import Content
+
+from unique_toolkit.agentic.postprocessor.postprocessor_manager import (
+    ResponsesApiPostprocessor,
+)
+from unique_toolkit.agentic.tools.config import get_configuration_dict
+from unique_toolkit.content.schemas import ContentReference
+from unique_toolkit.content.service import ContentService
+from unique_toolkit.language_model.schemas import ResponsesLanguageModelStreamResponse
+
+logger = logging.getLogger(__name__)
+
+
+class DisplayCodeInterpreterFilesPostProcessorConfig(BaseModel):
+    model_config = get_configuration_dict()
+    upload_scope_id: str
+
+
+class DisplayCodeInterpreterFilesPostProcessor(
+    ResponsesApiPostprocessor,
+):
+    def __init__(
+        self,
+        client: AsyncOpenAI,
+        content_service: ContentService,
+        config: DisplayCodeInterpreterFilesPostProcessorConfig,
+    ) -> None:
+        super().__init__(self.__class__.__name__)
+        self._content_service = content_service
+        self._config = config
+        self._client = client
+        self._content_map = {}
+
+    @override
+    async def run(self, loop_response: ResponsesLanguageModelStreamResponse) -> None:
+        logger.info("Fetching and adding code interpreter files to the response")
+
+        container_files = loop_response.container_files
+        logger.info("Found %s container files", len(container_files))
+
+        self._content_map = {}
+        for container_file in container_files:
+            logger.info("Fetching file content for %s", container_file.filename)
+            file_content = await self._client.containers.files.content.retrieve(
+                container_id=container_file.container_id, file_id=container_file.file_id
+            )
+
+            logger.info(
+                "Uploading file content for %s to knowledge base",
+                container_file.filename,
+            )
+            content = self._content_service.upload_content_from_bytes(
+                content=file_content.content,
+                content_name=container_file.filename,
+                skip_ingestion=True,
+                mime_type=guess_type(container_file.filename)[0] or "text/plain",
+                scope_id=self._config.upload_scope_id,
+            )
+            self._content_map[container_file.filename] = content
+
+    @override
+    def apply_postprocessing_to_response(
+        self, loop_response: ResponsesLanguageModelStreamResponse
+    ) -> bool:
+        ref_number = _get_next_ref_number(loop_response.message.references)
+        changed = False
+        # images
+        for filename, content in self._content_map.items():
+            # Images
+            loop_response.message.text, replaced = _replace_container_image_citation(
+                text=loop_response.message.text, filename=filename, content=content
+            )
+            changed |= replaced
+
+            # Files
+            loop_response.message.text, replaced = _replace_container_file_citation(
+                text=loop_response.message.text,
+                filename=filename,
+                ref_number=ref_number,
+            )
+            changed |= replaced
+            if replaced:
+                loop_response.message.references.append(
+                    ContentReference(
+                        sequence_number=ref_number,
+                        source_id=content.id,
+                        source="node-ingestion-chunks",
+                        url=f"unique://content/{content.id}",
+                        name=filename,
+                    )
+                )
+                ref_number += 1
+        return changed
+
+    @override
+    async def remove_from_text(self, text) -> str:
+        return text
+
+
+def _get_next_ref_number(references: list[ContentReference]) -> int:
+    max_ref_number = 0
+    for ref in references:
+        max_ref_number = max(max_ref_number, ref.sequence_number)
+    return max_ref_number + 1
+
+
+def _replace_container_image_citation(
+    text: str, filename: str, content: Content
+) -> tuple[str, bool]:
+    image_markdown = rf"!\[.*?\]\(sandbox:/mnt/data/{re.escape(filename)}\)"
+
+    if not re.search(image_markdown, text):
+        logger.info("No image markdown found for %s", filename)
+        return text, False
+
+    logger.info("Displaying image %s", filename)
+    return re.sub(
+        image_markdown,
+        f"![image](unique://content/{content.id})",
+        text,
+    ), True
+
+
+def _replace_container_file_citation(
+    text: str, filename: str, ref_number: int
+) -> tuple[str, bool]:
+    file_markdown = rf"\[.*?\]\(sandbox:/mnt/data/{re.escape(filename)}\)"
+
+    if not re.search(file_markdown, text):
+        logger.info("No file markdown found for %s", filename)
+        return text, False
+
+    logger.info("Displaying file %s", filename)
+    return re.sub(
+        file_markdown,
+        f"<sup>{ref_number}</sup>",
+        text,
+    ), True

unique_toolkit/agentic/responses_api/stream_handler.py

@@ -0,0 +1,15 @@
+from unique_toolkit.protocols.support import ResponsesSupportCompleteWithReferences
+from unique_toolkit.services.chat_service import ChatService
+
+
+class ResponsesStreamingHandler(ResponsesSupportCompleteWithReferences):
+    def __init__(self, chat_service: ChatService):
+        self._chat_service = chat_service
+
+    def complete_with_references(self, *args, **kwargs):
+        return self._chat_service.complete_responses_with_references(*args, **kwargs)
+
+    async def complete_with_references_async(self, *args, **kwargs):
+        return await self._chat_service.complete_responses_with_references_async(
+            *args, **kwargs
+        )

unique_toolkit/agentic/tools/factory.py

@@ -11,6 +11,10 @@ class ToolFactory:
     tool_map: dict[str, type[Tool]] = {}
     tool_config_map: dict[str, Callable] = {}
 
+    @classmethod
+    def register_tool_config(cls, tool_name: str, tool_config: type[BaseToolConfig]):
+        cls.tool_config_map[tool_name] = tool_config
+
     @classmethod
     def register_tool(
         cls,

unique_toolkit/agentic/tools/openai_builtin/__init__.py

@@ -0,0 +1,11 @@
+from unique_toolkit.agentic.tools.openai_builtin.code_interpreter import (
+    OpenAICodeInterpreterConfig,
+    OpenAICodeInterpreterTool,
+)
+from unique_toolkit.agentic.tools.openai_builtin.manager import OpenAIBuiltInToolManager
+
+__all__ = [
+    "OpenAIBuiltInToolManager",
+    "OpenAICodeInterpreterTool",
+    "OpenAICodeInterpreterConfig",
+]

unique_toolkit/agentic/tools/openai_builtin/base.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from typing import Generic, TypeVar
+
+from openai.types.responses.tool_param import CodeInterpreter
+
+from unique_toolkit.agentic.tools.schemas import ToolPrompts
+
+
+class OpenAIBuiltInToolName(StrEnum):
+    CODE_INTERPRETER = "code_interpreter"
+
+
+BuiltInToolType = CodeInterpreter  # Add other tool types when needed
+ToolType = TypeVar("ToolType", bound=BuiltInToolType)
+
+
+class OpenAIBuiltInTool(ABC, Generic[ToolType]):
+    @property
+    @abstractmethod
+    def name(self) -> OpenAIBuiltInToolName:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def tool_description(self) -> BuiltInToolType:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_tool_prompts(self) -> ToolPrompts:
+        raise NotImplementedError()

unique_toolkit/agentic/tools/openai_builtin/code_interpreter/__init__.py

@@ -0,0 +1,8 @@
+from unique_toolkit.agentic.tools.openai_builtin.code_interpreter.config import (
+    OpenAICodeInterpreterConfig,
+)
+from unique_toolkit.agentic.tools.openai_builtin.code_interpreter.service import (
+    OpenAICodeInterpreterTool,
+)
+
+__all__ = ["OpenAICodeInterpreterConfig", "OpenAICodeInterpreterTool"]

unique_toolkit/agentic/tools/openai_builtin/code_interpreter/config.py

@@ -0,0 +1,57 @@
+from pydantic import Field
+
+from unique_toolkit.agentic.tools.factory import ToolFactory
+from unique_toolkit.agentic.tools.openai_builtin.base import (
+    OpenAIBuiltInToolName,
+)
+from unique_toolkit.agentic.tools.schemas import BaseToolConfig
+
+DEFAULT_TOOL_DESCRIPTION = "Use this tool to run python code, e.g to generate plots, process excel files, perform calculations, etc."
+
+DEFAULT_TOOL_DESCRIPTION_FOR_SYSTEM_PROMPT = """
+Use this tool to run python code, e.g to generate plots, process excel files, perform calculations, etc.
+Instructions:
+- All files uploaded to the chat are available in the code interpreter under the path `/mnt/data/<filename>
+- All files generated through code should be saved in the `/mnt/data` folder
+
+Instructions for displaying images and files in the chat:
+Once files are generated in the `/mnt/data` folder you MUST reference them in the chat using markdown syntax in order to display them in the chat.
+
+- If you want to display an image, use the following syntax: `![Image Name](sandbox:/mnt/data/<filename>)`
+    - Images will be converted and shown in the chat.
+    - Do NOT display an extra download link for images a part from the markdown above.
+    - Not using markdown syntax will FAIL to show images to the user. 
+    - YOU MUST use the syntax above to display images, otherwise the image will not be displayed in the chat.
+    - For displaying a link to a file, use the following syntax: `[filename](sandbox:/mnt/data/<filename>)`
+        - Files are converted to references the user can click on to download the file
+
+You MUST always use this syntax, otherwise the files will not be displayed in the chat.
+""".strip()
+
+DEFAULT_TOOL_FORMAT_INFORMATION_FOR_SYSTEM_PROMPT = ""
+
+DEFAULT_TOOL_FORMAT_INFORMATION_FOR_USER_PROMPT = ""
+
+DEFAULT_TOOL_DESCRIPTION_FOR_USER_PROMPT = ""
+
+
+class OpenAICodeInterpreterConfig(BaseToolConfig):
+    upload_files_in_chat: bool = Field(default=True)
+
+    tool_description: str = DEFAULT_TOOL_DESCRIPTION
+    tool_description_for_system_prompt: str = DEFAULT_TOOL_DESCRIPTION_FOR_SYSTEM_PROMPT
+    tool_format_information_for_system_prompt: str = (
+        DEFAULT_TOOL_FORMAT_INFORMATION_FOR_SYSTEM_PROMPT
+    )
+    tool_description_for_user_prompt: str = DEFAULT_TOOL_DESCRIPTION_FOR_USER_PROMPT
+    tool_format_information_for_user_prompt: str = (
+        DEFAULT_TOOL_FORMAT_INFORMATION_FOR_USER_PROMPT
+    )
+
+    expires_after_minutes: int = 20
+    use_auto_container: bool = False
+
+
+ToolFactory.register_tool_config(
+    OpenAIBuiltInToolName.CODE_INTERPRETER, OpenAICodeInterpreterConfig
+)