llama-stack 0.3.5__py3-none-any.whl → 0.4.1__py3-none-any.whl

llama_stack/providers/inline/agents/meta_reference/responses/utils.py

@@ -5,15 +5,36 @@
 # the root directory of this source tree.
 
 import asyncio
+import base64
+import mimetypes
 import re
 import uuid
+from collections.abc import Sequence
 
-from llama_stack.apis.agents.agents import ResponseGuardrailSpec
-from llama_stack.apis.agents.openai_responses import (
+from llama_stack_api import (
+    Files,
+    OpenAIAssistantMessageParam,
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartParam,
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionToolCall,
+    OpenAIChatCompletionToolCallFunction,
+    OpenAIChoice,
+    OpenAIDeveloperMessageParam,
+    OpenAIFile,
+    OpenAIFileFile,
+    OpenAIImageURL,
+    OpenAIJSONSchema,
+    OpenAIMessageParam,
     OpenAIResponseAnnotationFileCitation,
+    OpenAIResponseFormatJSONObject,
+    OpenAIResponseFormatJSONSchema,
+    OpenAIResponseFormatParam,
+    OpenAIResponseFormatText,
     OpenAIResponseInput,
     OpenAIResponseInputFunctionToolCallOutput,
     OpenAIResponseInputMessageContent,
+    OpenAIResponseInputMessageContentFile,
     OpenAIResponseInputMessageContentImage,
     OpenAIResponseInputMessageContentText,
     OpenAIResponseInputTool,
@@ -22,32 +43,58 @@ from llama_stack.apis.agents.openai_responses import (
     OpenAIResponseMessage,
     OpenAIResponseOutputMessageContent,
     OpenAIResponseOutputMessageContentOutputText,
+    OpenAIResponseOutputMessageFileSearchToolCall,
     OpenAIResponseOutputMessageFunctionToolCall,
     OpenAIResponseOutputMessageMCPCall,
     OpenAIResponseOutputMessageMCPListTools,
+    OpenAIResponseOutputMessageWebSearchToolCall,
     OpenAIResponseText,
-)
-from llama_stack.apis.inference import (
-    OpenAIAssistantMessageParam,
-    OpenAIChatCompletionContentPartImageParam,
-    OpenAIChatCompletionContentPartParam,
-    OpenAIChatCompletionContentPartTextParam,
-    OpenAIChatCompletionToolCall,
-    OpenAIChatCompletionToolCallFunction,
-    OpenAIChoice,
-    OpenAIDeveloperMessageParam,
-    OpenAIImageURL,
-    OpenAIJSONSchema,
-    OpenAIMessageParam,
-    OpenAIResponseFormatJSONObject,
-    OpenAIResponseFormatJSONSchema,
-    OpenAIResponseFormatParam,
-    OpenAIResponseFormatText,
     OpenAISystemMessageParam,
     OpenAIToolMessageParam,
     OpenAIUserMessageParam,
+    ResponseGuardrailSpec,
+    Safety,
 )
-from llama_stack.apis.safety import Safety
+
+
+async def extract_bytes_from_file(file_id: str, files_api: Files) -> bytes:
+    """
+    Extract raw bytes from file using the Files API.
+
+    :param file_id: The file identifier (e.g., "file-abc123")
+    :param files_api: Files API instance
+    :returns: Raw file content as bytes
+    :raises: ValueError if file cannot be retrieved
+    """
+    try:
+        response = await files_api.openai_retrieve_file_content(file_id)
+        return bytes(response.body)
+    except Exception as e:
+        raise ValueError(f"Failed to retrieve file content for file_id '{file_id}': {str(e)}") from e
+
+
+def generate_base64_ascii_text_from_bytes(raw_bytes: bytes) -> str:
+    """
+    Converts raw binary bytes into a safe ASCII text representation for URLs
+
+    :param raw_bytes: the actual bytes that represents file content
+    :returns: string of utf-8 characters
+    """
+    return base64.b64encode(raw_bytes).decode("utf-8")
+
+
+def construct_data_url(ascii_text: str, mime_type: str | None) -> str:
+    """
+    Construct data url with decoded data inside
+
+    :param ascii_text: ASCII content
+    :param mime_type: MIME type of file
+    :returns: data url string (eg. data:image/png,base64,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E)
+    """
+    if not mime_type:
+        mime_type = "application/octet-stream"
+
+    return f"data:{mime_type};base64,{ascii_text}"
 
 
 async def convert_chat_choice_to_response_message(
@@ -68,36 +115,107 @@ async def convert_chat_choice_to_response_message(
         )
 
     annotations, clean_text = _extract_citations_from_text(output_content, citation_files or {})
+    logprobs = choice.logprobs.content if choice.logprobs and choice.logprobs.content else None
 
     return OpenAIResponseMessage(
         id=message_id or f"msg_{uuid.uuid4()}",
-        content=[OpenAIResponseOutputMessageContentOutputText(text=clean_text, annotations=annotations)],
+        content=[
+            OpenAIResponseOutputMessageContentOutputText(
+                text=clean_text,
+                annotations=list(annotations),
+                logprobs=logprobs,
+            )
+        ],
         status="completed",
         role="assistant",
     )
 
 
 async def convert_response_content_to_chat_content(
-    content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]),
+    content: str | Sequence[OpenAIResponseInputMessageContent | OpenAIResponseOutputMessageContent],
+    files_api: Files | None,
 ) -> str | list[OpenAIChatCompletionContentPartParam]:
     """
     Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts.
 
     The content schemas of each API look similar, but are not exactly the same.
+
+    :param content: The content to convert
+    :param files_api: Files API for resolving file_id to raw file content (required if content contains files/images)
     """
     if isinstance(content, str):
         return content
 
-    converted_parts = []
+    # Type with union to avoid list invariance issues
+    converted_parts: list[OpenAIChatCompletionContentPartParam] = []
     for content_part in content:
         if isinstance(content_part, OpenAIResponseInputMessageContentText):
             converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
         elif isinstance(content_part, OpenAIResponseOutputMessageContentOutputText):
             converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text))
         elif isinstance(content_part, OpenAIResponseInputMessageContentImage):
+            detail = content_part.detail
+            image_mime_type = None
             if content_part.image_url:
-                image_url = OpenAIImageURL(url=content_part.image_url, detail=content_part.detail)
+                image_url = OpenAIImageURL(url=content_part.image_url, detail=detail)
                 converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url))
+            elif content_part.file_id:
+                if files_api is None:
+                    raise ValueError("file_ids are not supported by this implementation of the Stack")
+                image_file_response = await files_api.openai_retrieve_file(content_part.file_id)
+                if image_file_response.filename:
+                    image_mime_type, _ = mimetypes.guess_type(image_file_response.filename)
+                raw_image_bytes = await extract_bytes_from_file(content_part.file_id, files_api)
+                ascii_text = generate_base64_ascii_text_from_bytes(raw_image_bytes)
+                image_data_url = construct_data_url(ascii_text, image_mime_type)
+                image_url = OpenAIImageURL(url=image_data_url, detail=detail)
+                converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url))
+            else:
+                raise ValueError(
+                    f"Image content must have either 'image_url' or 'file_id'. "
+                    f"Got image_url={content_part.image_url}, file_id={content_part.file_id}"
+                )
+        elif isinstance(content_part, OpenAIResponseInputMessageContentFile):
+            resolved_file_data = None
+            file_data = content_part.file_data
+            file_id = content_part.file_id
+            file_url = content_part.file_url
+            filename = content_part.filename
+            file_mime_type = None
+            if not any([file_data, file_id, file_url]):
+                raise ValueError(
+                    f"File content must have at least one of 'file_data', 'file_id', or 'file_url'. "
+                    f"Got file_data={file_data}, file_id={file_id}, file_url={file_url}"
+                )
+            if file_id:
+                if files_api is None:
+                    raise ValueError("file_ids are not supported by this implementation of the Stack")
+
+                file_response = await files_api.openai_retrieve_file(file_id)
+                if not filename:
+                    filename = file_response.filename
+                file_mime_type, _ = mimetypes.guess_type(file_response.filename)
+                raw_file_bytes = await extract_bytes_from_file(file_id, files_api)
+                ascii_text = generate_base64_ascii_text_from_bytes(raw_file_bytes)
+                resolved_file_data = construct_data_url(ascii_text, file_mime_type)
+            elif file_data:
+                if file_data.startswith("data:"):
+                    resolved_file_data = file_data
+                else:
+                    # Raw base64 data, wrap in data URL format
+                    if filename:
+                        file_mime_type, _ = mimetypes.guess_type(filename)
+                    resolved_file_data = construct_data_url(file_data, file_mime_type)
+            elif file_url:
+                resolved_file_data = file_url
+            converted_parts.append(
+                OpenAIFile(
+                    file=OpenAIFileFile(
+                        file_data=resolved_file_data,
+                        filename=filename,
+                    )
+                )
+            )
         elif isinstance(content_part, str):
             converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part))
         else:
@@ -110,12 +228,14 @@ async def convert_response_content_to_chat_content(
 async def convert_response_input_to_chat_messages(
     input: str | list[OpenAIResponseInput],
     previous_messages: list[OpenAIMessageParam] | None = None,
+    files_api: Files | None = None,
 ) -> list[OpenAIMessageParam]:
     """
     Convert the input from an OpenAI Response API request into OpenAI Chat Completion messages.
 
     :param input: The input to convert
     :param previous_messages: Optional previous messages to check for function_call references
+    :param files_api: Files API for resolving file_id to raw file content (optional, required for file/image content)
     """
     messages: list[OpenAIMessageParam] = []
     if isinstance(input, list):
@@ -158,22 +278,31 @@ async def convert_response_input_to_chat_messages(
                     ),
                 )
                 messages.append(OpenAIAssistantMessageParam(tool_calls=[tool_call]))
+                # Output can be None, use empty string as fallback
+                output_content = input_item.output if input_item.output is not None else ""
                 messages.append(
                     OpenAIToolMessageParam(
-                        content=input_item.output,
+                        content=output_content,
                         tool_call_id=input_item.id,
                     )
                 )
             elif isinstance(input_item, OpenAIResponseOutputMessageMCPListTools):
                 # the tool list will be handled separately
                 pass
+            elif isinstance(
+                input_item,
+                OpenAIResponseOutputMessageWebSearchToolCall | OpenAIResponseOutputMessageFileSearchToolCall,
+            ):
+                # these tool calls are tracked internally but not converted to chat messages
+                pass
             elif isinstance(input_item, OpenAIResponseMCPApprovalRequest) or isinstance(
                 input_item, OpenAIResponseMCPApprovalResponse
             ):
                 # these are handled by the responses impl itself and not pass through to chat completions
                 pass
-            else:
-                content = await convert_response_content_to_chat_content(input_item.content)
+            elif isinstance(input_item, OpenAIResponseMessage):
+                # Narrow type to OpenAIResponseMessage which has content and role attributes
+                content = await convert_response_content_to_chat_content(input_item.content, files_api)
                 message_type = await get_message_type_by_role(input_item.role)
                 if message_type is None:
                     raise ValueError(
@@ -191,7 +320,8 @@ async def convert_response_input_to_chat_messages(
                         last_user_content = getattr(last_user_msg, "content", None)
                         if last_user_content == content:
                             continue  # Skip duplicate user message
-                messages.append(message_type(content=content))
+                # Dynamic message type call - different message types have different content expectations
+                messages.append(message_type(content=content))  # type: ignore[call-arg,arg-type]
         if len(tool_call_results):
             # Check if unpaired function_call_outputs reference function_calls from previous messages
             if previous_messages:
@@ -237,8 +367,11 @@ async def convert_response_text_to_chat_response_format(
     if text.format["type"] == "json_object":
         return OpenAIResponseFormatJSONObject()
     if text.format["type"] == "json_schema":
+        # Assert name exists for json_schema format
+        assert text.format.get("name"), "json_schema format requires a name"
+        schema_name: str = text.format["name"]  # type: ignore[assignment]
         return OpenAIResponseFormatJSONSchema(
-            json_schema=OpenAIJSONSchema(name=text.format["name"], schema=text.format["schema"])
+            json_schema=OpenAIJSONSchema(name=schema_name, schema=text.format["schema"])
         )
     raise ValueError(f"Unsupported text format: {text.format}")
 
@@ -251,7 +384,7 @@ async def get_message_type_by_role(role: str) -> type[OpenAIMessageParam] | None
         "assistant": OpenAIAssistantMessageParam,
         "developer": OpenAIDeveloperMessageParam,
     }
-    return role_to_type.get(role)
+    return role_to_type.get(role)  # type: ignore[return-value]  # Pydantic models use ModelMetaclass
 
 
 def _extract_citations_from_text(
@@ -313,14 +446,19 @@ def is_function_tool_call(
     return False
 
 
-async def run_guardrails(safety_api: Safety, messages: str, guardrail_ids: list[str]) -> str | None:
+async def run_guardrails(safety_api: Safety | None, messages: str, guardrail_ids: list[str]) -> str | None:
     """Run guardrails against messages and return violation message if blocked."""
     if not messages:
         return None
 
+    # If safety API is not available, skip guardrails
+    if safety_api is None:
+        return None
+
     # Look up shields to get their provider_resource_id (actual model ID)
     model_ids = []
-    shields_list = await safety_api.routing_table.list_shields()
+    # TODO: list_shields not in Safety interface but available at runtime via API routing
+    shields_list = await safety_api.routing_table.list_shields()  # type: ignore[attr-defined]
 
     for guardrail_id in guardrail_ids:
         matching_shields = [shield for shield in shields_list.data if shield.identifier == guardrail_id]
@@ -337,7 +475,9 @@ async def run_guardrails(safety_api: Safety, messages: str, guardrail_ids: list[
         for result in response.results:
             if result.flagged:
                 message = result.user_message or "Content blocked by safety guardrails"
-                flagged_categories = [cat for cat, flagged in result.categories.items() if flagged]
+                flagged_categories = (
+                    [cat for cat, flagged in result.categories.items() if flagged] if result.categories else []
+                )
                 violation_type = result.metadata.get("violation_type", []) if result.metadata else []
 
                 if flagged_categories:
@@ -347,6 +487,9 @@ async def run_guardrails(safety_api: Safety, messages: str, guardrail_ids: list[
 
                 return message
 
+    # No violations found
+    return None
+
 
 def extract_guardrail_ids(guardrails: list | None) -> list[str]:
     """Extract guardrail IDs from guardrails parameter, handling both string IDs and ResponseGuardrailSpec objects."""
@@ -363,3 +506,28 @@ def extract_guardrail_ids(guardrails: list | None) -> list[str]:
             raise ValueError(f"Unknown guardrail format: {guardrail}, expected str or ResponseGuardrailSpec")
 
     return guardrail_ids
+
+
+def convert_mcp_tool_choice(
+    chat_tool_names: list[str],
+    server_label: str | None = None,
+    server_label_to_tools: dict[str, list[str]] | None = None,
+    tool_name: str | None = None,
+) -> dict[str, str] | list[dict[str, str]]:
+    """Convert a responses tool choice of type mcp to a chat completions compatible function tool choice."""
+
+    if tool_name:
+        if tool_name not in chat_tool_names:
+            return None
+        return {"type": "function", "function": {"name": tool_name}}
+
+    elif server_label and server_label_to_tools:
+        # no tool name specified, so we need to enforce an allowed_tools with the function tools derived only from the given server label
+        # Use reverse mapping for lookup by server_label
+        # This already accounts for allowed_tools restrictions applied during _process_mcp_tool
+        tool_names = server_label_to_tools.get(server_label, [])
+        if not tool_names:
+            return None
+        matching_tools = [{"type": "function", "function": {"name": tool_name}} for tool_name in tool_names]
+        return matching_tools
+    return []

llama_stack/providers/inline/agents/meta_reference/safety.py

@@ -6,10 +6,8 @@
 
 import asyncio
 
-from llama_stack.apis.inference import Message
-from llama_stack.apis.safety import Safety, SafetyViolation, ViolationLevel
 from llama_stack.log import get_logger
-from llama_stack.providers.utils.telemetry import tracing
+from llama_stack_api import OpenAIMessageParam, Safety, SafetyViolation, ViolationLevel
 
 log = get_logger(name=__name__, category="agents::meta_reference")
 
@@ -31,16 +29,13 @@ class ShieldRunnerMixin:
         self.input_shields = input_shields
         self.output_shields = output_shields
 
-    async def run_multiple_shields(self, messages: list[Message], identifiers: list[str]) -> None:
-        async def run_shield_with_span(identifier: str):
-            async with tracing.span(f"run_shield_{identifier}"):
-                return await self.safety_api.run_shield(
-                    shield_id=identifier,
-                    messages=messages,
-                    params={},
-                )
-
-        responses = await asyncio.gather(*[run_shield_with_span(identifier) for identifier in identifiers])
+    async def run_multiple_shields(self, messages: list[OpenAIMessageParam], identifiers: list[str]) -> None:
+        responses = await asyncio.gather(
+            *[
+                self.safety_api.run_shield(shield_id=identifier, messages=messages, params={})
+                for identifier in identifiers
+            ]
+        )
         for identifier, response in zip(identifiers, responses, strict=False):
             if not response.violation:
                 continue

llama_stack/providers/inline/batches/reference/__init__.py

@@ -6,11 +6,9 @@
 
 from typing import Any
 
-from llama_stack.apis.files import Files
-from llama_stack.apis.inference import Inference
-from llama_stack.apis.models import Models
 from llama_stack.core.datatypes import AccessRule, Api
-from llama_stack.providers.utils.kvstore import kvstore_impl
+from llama_stack.core.storage.kvstore import kvstore_impl
+from llama_stack_api import Files, Inference, Models
 
 from .batches import ReferenceBatchesImpl
 from .config import ReferenceBatchesImplConfig