llama-stack 0.4.3__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack/providers/inline/agents/meta_reference/responses/streaming.py

@@ -4,6 +4,7 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
+import time
 import uuid
 from collections.abc import AsyncIterator
 from typing import Any
@@ -16,6 +17,7 @@ from llama_stack.providers.utils.inference.prompt_adapter import interleaved_con
 from llama_stack_api import (
     AllowedToolsFilter,
     ApprovalFilter,
+    Connectors,
     Inference,
     MCPListToolsTool,
     ModelNotFoundError,
@@ -30,6 +32,7 @@ from llama_stack_api import (
     OpenAIChatCompletionToolChoiceFunctionTool,
     OpenAIChoice,
     OpenAIChoiceLogprobs,
+    OpenAIFinishReason,
     OpenAIMessageParam,
     OpenAIResponseContentPartOutputText,
     OpenAIResponseContentPartReasoningText,
@@ -77,6 +80,7 @@ from llama_stack_api import (
     OpenAIResponseOutputMessageMCPListTools,
     OpenAIResponseOutputMessageWebSearchToolCall,
     OpenAIResponsePrompt,
+    OpenAIResponseReasoning,
     OpenAIResponseText,
     OpenAIResponseUsage,
     OpenAIResponseUsageInputTokensDetails,
@@ -133,11 +137,15 @@ class StreamingResponseOrchestrator:
         instructions: str | None,
         safety_api: Safety | None,
         guardrail_ids: list[str] | None = None,
+        connectors_api: Connectors | None = None,
         prompt: OpenAIResponsePrompt | None = None,
         parallel_tool_calls: bool | None = None,
         max_tool_calls: int | None = None,
+        reasoning: OpenAIResponseReasoning | None = None,
+        max_output_tokens: int | None = None,
         metadata: dict[str, str] | None = None,
         include: list[ResponseItemInclude] | None = None,
+        store: bool | None = True,
     ):
         self.inference_api = inference_api
         self.ctx = ctx
@@ -147,6 +155,7 @@ class StreamingResponseOrchestrator:
         self.max_infer_iters = max_infer_iters
         self.tool_executor = tool_executor
         self.safety_api = safety_api
+        self.connectors_api = connectors_api
         self.guardrail_ids = guardrail_ids or []
         self.prompt = prompt
         # System message that is inserted into the model's context
@@ -155,8 +164,13 @@ class StreamingResponseOrchestrator:
         self.parallel_tool_calls = parallel_tool_calls
         # Max number of total calls to built-in tools that can be processed in a response
         self.max_tool_calls = max_tool_calls
+        self.reasoning = reasoning
+        # An upper bound for the number of tokens that can be generated for a response
+        self.max_output_tokens = max_output_tokens
         self.metadata = metadata
+        self.store = store
         self.include = include
+        self.store = bool(store) if store is not None else True
         self.sequence_number = 0
         # Store MCP tool mapping that gets built during tool processing
         self.mcp_tool_to_server: dict[str, OpenAIResponseInputToolMCP] = (
@@ -179,6 +193,8 @@ class StreamingResponseOrchestrator:
         self.violation_detected = False
         # Track total calls made to built-in tools
         self.accumulated_builtin_tool_calls = 0
+        # Track total output tokens generated across inference calls
+        self.accumulated_builtin_output_tokens = 0
 
     async def _create_refusal_response(self, violation_message: str) -> OpenAIResponseObjectStream:
         """Create a refusal response to replace streaming content."""
@@ -191,7 +207,9 @@ class StreamingResponseOrchestrator:
             model=self.ctx.model,
             status="completed",
             output=[OpenAIResponseMessage(role="assistant", content=[refusal_content], type="message")],
+            max_output_tokens=self.max_output_tokens,
             metadata=self.metadata,
+            store=self.store,
         )
 
         return OpenAIResponseObjectStreamResponseCompleted(response=refusal_response)
@@ -212,8 +230,10 @@ class StreamingResponseOrchestrator:
         *,
         error: OpenAIResponseError | None = None,
     ) -> OpenAIResponseObject:
+        completed_at = int(time.time()) if status == "completed" else None
         return OpenAIResponseObject(
             created_at=self.created_at,
+            completed_at=completed_at,
             id=self.response_id,
             model=self.ctx.model,
             object="response",
@@ -228,7 +248,10 @@ class StreamingResponseOrchestrator:
             prompt=self.prompt,
             parallel_tool_calls=self.parallel_tool_calls,
             max_tool_calls=self.max_tool_calls,
+            reasoning=self.reasoning,
+            max_output_tokens=self.max_output_tokens,
             metadata=self.metadata,
+            store=self.store,
         )
 
     async def create_response(self) -> AsyncIterator[OpenAIResponseObjectStream]:
@@ -292,6 +315,22 @@ class StreamingResponseOrchestrator:
 
         try:
             while True:
+                if (
+                    self.max_output_tokens is not None
+                    and self.accumulated_builtin_output_tokens >= self.max_output_tokens
+                ):
+                    logger.info(
+                        "Skipping inference call since max_output_tokens reached: "
+                        f"{self.accumulated_builtin_output_tokens}/{self.max_output_tokens}"
+                    )
+                    final_status = "incomplete"
+                    break
+
+                remaining_output_tokens = (
+                    self.max_output_tokens - self.accumulated_builtin_output_tokens
+                    if self.max_output_tokens is not None
+                    else None
+                )
                 # Text is the default response format for chat completion so don't need to pass it
                 # (some providers don't support non-empty response_format when tools are present)
                 response_format = (
@@ -311,6 +350,11 @@ class StreamingResponseOrchestrator:
                     True if self.include and ResponseItemInclude.message_output_text_logprobs in self.include else None
                 )
 
+                # In OpenAI, parallel_tool_calls is only allowed when 'tools' are specified.
+                effective_parallel_tool_calls = (
+                    self.parallel_tool_calls if effective_tools is not None and len(effective_tools) > 0 else None
+                )
+
                 params = OpenAIChatCompletionRequestWithExtraBody(
                     model=self.ctx.model,
                     messages=messages,
@@ -324,6 +368,9 @@ class StreamingResponseOrchestrator:
                         "include_usage": True,
                     },
                     logprobs=logprobs,
+                    parallel_tool_calls=effective_parallel_tool_calls,
+                    reasoning_effort=self.reasoning.effort if self.reasoning else None,
+                    max_completion_tokens=remaining_output_tokens,
                 )
                 completion_result = await self.inference_api.openai_chat_completion(params)
 
@@ -480,23 +527,24 @@ class StreamingResponseOrchestrator:
         if not chunk.usage:
             return
 
+        self.accumulated_builtin_output_tokens += chunk.usage.completion_tokens
+
         if self.accumulated_usage is None:
             # Convert from chat completion format to response format
             self.accumulated_usage = OpenAIResponseUsage(
                 input_tokens=chunk.usage.prompt_tokens,
                 output_tokens=chunk.usage.completion_tokens,
                 total_tokens=chunk.usage.total_tokens,
-                input_tokens_details=(
-                    OpenAIResponseUsageInputTokensDetails(cached_tokens=chunk.usage.prompt_tokens_details.cached_tokens)
-                    if chunk.usage.prompt_tokens_details
-                    else None
+                input_tokens_details=OpenAIResponseUsageInputTokensDetails(
+                    cached_tokens=chunk.usage.prompt_tokens_details.cached_tokens
+                    if chunk.usage.prompt_tokens_details and chunk.usage.prompt_tokens_details.cached_tokens is not None
+                    else 0
                 ),
-                output_tokens_details=(
-                    OpenAIResponseUsageOutputTokensDetails(
-                        reasoning_tokens=chunk.usage.completion_tokens_details.reasoning_tokens
-                    )
+                output_tokens_details=OpenAIResponseUsageOutputTokensDetails(
+                    reasoning_tokens=chunk.usage.completion_tokens_details.reasoning_tokens
                     if chunk.usage.completion_tokens_details
-                    else None
+                    and chunk.usage.completion_tokens_details.reasoning_tokens is not None
+                    else 0
                 ),
             )
         else:
@@ -506,17 +554,16 @@ class StreamingResponseOrchestrator:
                 output_tokens=self.accumulated_usage.output_tokens + chunk.usage.completion_tokens,
                 total_tokens=self.accumulated_usage.total_tokens + chunk.usage.total_tokens,
                 # Use latest non-null details
-                input_tokens_details=(
-                    OpenAIResponseUsageInputTokensDetails(cached_tokens=chunk.usage.prompt_tokens_details.cached_tokens)
-                    if chunk.usage.prompt_tokens_details
-                    else self.accumulated_usage.input_tokens_details
+                input_tokens_details=OpenAIResponseUsageInputTokensDetails(
+                    cached_tokens=chunk.usage.prompt_tokens_details.cached_tokens
+                    if chunk.usage.prompt_tokens_details and chunk.usage.prompt_tokens_details.cached_tokens is not None
+                    else self.accumulated_usage.input_tokens_details.cached_tokens
                 ),
-                output_tokens_details=(
-                    OpenAIResponseUsageOutputTokensDetails(
-                        reasoning_tokens=chunk.usage.completion_tokens_details.reasoning_tokens
-                    )
+                output_tokens_details=OpenAIResponseUsageOutputTokensDetails(
+                    reasoning_tokens=chunk.usage.completion_tokens_details.reasoning_tokens
                     if chunk.usage.completion_tokens_details
-                    else self.accumulated_usage.output_tokens_details
+                    and chunk.usage.completion_tokens_details.reasoning_tokens is not None
+                    else self.accumulated_usage.output_tokens_details.reasoning_tokens
                 ),
             )
 
@@ -652,7 +699,7 @@ class StreamingResponseOrchestrator:
         chat_response_tool_calls: dict[int, OpenAIChatCompletionToolCall] = {}
         chunk_created = 0
         chunk_model = ""
-        chunk_finish_reason = ""
+        chunk_finish_reason: OpenAIFinishReason = "stop"
         chat_response_logprobs = []
 
         # Create a placeholder message item for delta events
@@ -744,9 +791,9 @@ class StreamingResponseOrchestrator:
                     chunk_finish_reason = chunk_choice.finish_reason
 
                 # Handle reasoning content if present (non-standard field for o1/o3 models)
-                if hasattr(chunk_choice.delta, "reasoning_content") and chunk_choice.delta.reasoning_content:
+                if hasattr(chunk_choice.delta, "reasoning") and chunk_choice.delta.reasoning:
                     async for event in self._handle_reasoning_content_chunk(
-                        reasoning_content=chunk_choice.delta.reasoning_content,
+                        reasoning_content=chunk_choice.delta.reasoning,
                         reasoning_part_emitted=reasoning_part_emitted,
                         reasoning_content_index=reasoning_content_index,
                         message_item_id=message_item_id,
@@ -758,7 +805,7 @@ class StreamingResponseOrchestrator:
                         else:
                             yield event
                     reasoning_part_emitted = True
-                    reasoning_text_accumulated.append(chunk_choice.delta.reasoning_content)
+                    reasoning_text_accumulated.append(chunk_choice.delta.reasoning)
 
                 # Handle refusal content if present
                 if chunk_choice.delta.refusal:
@@ -1175,6 +1222,9 @@ class StreamingResponseOrchestrator:
         """Process an MCP tool configuration and emit appropriate streaming events."""
         from llama_stack.providers.utils.tools.mcp import list_mcp_tools
 
+        # Resolve connector_id to server_url if provided
+        mcp_tool = await resolve_mcp_connector_id(mcp_tool, self.connectors_api)
+
         # Emit mcp_list_tools.in_progress
         self.sequence_number += 1
         yield OpenAIResponseObjectStreamResponseMcpListToolsInProgress(
@@ -1489,3 +1539,25 @@ async def _process_tool_choice(
                         tools=tool_choice,
                         mode="required",
                     )
+
+
+async def resolve_mcp_connector_id(
+    mcp_tool: OpenAIResponseInputToolMCP,
+    connectors_api: Connectors,
+) -> OpenAIResponseInputToolMCP:
+    """Resolve connector_id to server_url for an MCP tool.
+
+    If the mcp_tool has a connector_id but no server_url, this function
+    looks up the connector and populates the server_url from it.
+
+    Args:
+        mcp_tool: The MCP tool configuration to resolve
+        connectors_api: The connectors API for looking up connectors
+
+    Returns:
+        The mcp_tool with server_url populated (may be same instance if already set)
+    """
+    if mcp_tool.connector_id and not mcp_tool.server_url:
+        connector = await connectors_api.get_connector(mcp_tool.connector_id)
+        return mcp_tool.model_copy(update={"server_url": connector.url})
+    return mcp_tool

llama_stack/providers/inline/agents/meta_reference/responses/types.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel
 
 from llama_stack_api import (
     OpenAIChatCompletionToolCall,
+    OpenAIFinishReason,
     OpenAIMessageParam,
     OpenAIResponseFormatParam,
     OpenAIResponseInput,
@@ -52,7 +53,7 @@ class ChatCompletionResult:
     tool_calls: dict[int, OpenAIChatCompletionToolCall]
     created: int
     model: str
-    finish_reason: str
+    finish_reason: OpenAIFinishReason
     message_item_id: str  # For streaming events
     tool_call_item_ids: dict[int, str]  # For streaming events
     content_part_emitted: bool  # Tracking state

llama_stack/providers/inline/agents/meta_reference/responses/utils.py

@@ -53,6 +53,7 @@ from llama_stack_api import (
     OpenAIToolMessageParam,
     OpenAIUserMessageParam,
     ResponseGuardrailSpec,
+    RunModerationRequest,
     Safety,
 )
 
@@ -468,7 +469,9 @@ async def run_guardrails(safety_api: Safety | None, messages: str, guardrail_ids
         else:
             raise ValueError(f"No shield found with identifier '{guardrail_id}'")
 
-    guardrail_tasks = [safety_api.run_moderation(messages, model=model_id) for model_id in model_ids]
+    guardrail_tasks = [
+        safety_api.run_moderation(RunModerationRequest(input=messages, model=model_id)) for model_id in model_ids
+    ]
     responses = await asyncio.gather(*guardrail_tasks)
 
     for response in responses:

llama_stack/providers/inline/agents/meta_reference/safety.py

@@ -7,7 +7,7 @@
 import asyncio
 
 from llama_stack.log import get_logger
-from llama_stack_api import OpenAIMessageParam, Safety, SafetyViolation, ViolationLevel
+from llama_stack_api import OpenAIMessageParam, RunShieldRequest, Safety, SafetyViolation, ViolationLevel
 
 log = get_logger(name=__name__, category="agents::meta_reference")
 
@@ -32,7 +32,7 @@ class ShieldRunnerMixin:
     async def run_multiple_shields(self, messages: list[OpenAIMessageParam], identifiers: list[str]) -> None:
         responses = await asyncio.gather(
             *[
-                self.safety_api.run_shield(shield_id=identifier, messages=messages, params={})
+                self.safety_api.run_shield(RunShieldRequest(shield_id=identifier, messages=messages))
                 for identifier in identifiers
             ]
         )

llama_stack/providers/inline/batches/reference/batches.py

@@ -23,6 +23,7 @@ from llama_stack_api import (
     BatchObject,
     ConflictError,
     Files,
+    GetModelRequest,
     Inference,
     ListBatchesResponse,
     Models,
@@ -485,7 +486,7 @@ class ReferenceBatchesImpl(Batches):
 
                         if "model" in request_body and isinstance(request_body["model"], str):
                             try:
-                                await self.models_api.get_model(request_body["model"])
+                                await self.models_api.get_model(GetModelRequest(model_id=request_body["model"]))
                             except Exception:
                                 errors.append(
                                     BatchError(

llama_stack/providers/inline/eval/meta_reference/eval.py

@@ -13,19 +13,25 @@ from llama_stack.providers.utils.common.data_schema_validator import ColumnName
 from llama_stack_api import (
     Agents,
     Benchmark,
-    BenchmarkConfig,
     BenchmarksProtocolPrivate,
     DatasetIO,
     Datasets,
     Eval,
     EvaluateResponse,
+    EvaluateRowsRequest,
     Inference,
+    IterRowsRequest,
     Job,
+    JobCancelRequest,
+    JobResultRequest,
     JobStatus,
+    JobStatusRequest,
     OpenAIChatCompletionRequestWithExtraBody,
     OpenAICompletionRequestWithExtraBody,
     OpenAISystemMessageParam,
     OpenAIUserMessageParam,
+    RunEvalRequest,
+    ScoreRequest,
     Scoring,
 )
 
@@ -90,10 +96,9 @@ class MetaReferenceEvalImpl(
 
     async def run_eval(
         self,
-        benchmark_id: str,
-        benchmark_config: BenchmarkConfig,
+        request: RunEvalRequest,
     ) -> Job:
-        task_def = self.benchmarks[benchmark_id]
+        task_def = self.benchmarks[request.benchmark_id]
         dataset_id = task_def.dataset_id
         scoring_functions = task_def.scoring_functions
 
@@ -101,15 +106,18 @@ class MetaReferenceEvalImpl(
         # dataset_def = await self.datasets_api.get_dataset(dataset_id=dataset_id)
 
         all_rows = await self.datasetio_api.iterrows(
-            dataset_id=dataset_id,
-            limit=(-1 if benchmark_config.num_examples is None else benchmark_config.num_examples),
+            IterRowsRequest(
+                dataset_id=dataset_id,
+                limit=(-1 if request.benchmark_config.num_examples is None else request.benchmark_config.num_examples),
+            )
         )
-        res = await self.evaluate_rows(
-            benchmark_id=benchmark_id,
+        eval_rows_request = EvaluateRowsRequest(
+            benchmark_id=request.benchmark_id,
             input_rows=all_rows.data,
             scoring_functions=scoring_functions,
-            benchmark_config=benchmark_config,
+            benchmark_config=request.benchmark_config,
         )
+        res = await self.evaluate_rows(eval_rows_request)
 
         # TODO: currently needs to wait for generation before returning
         # need job scheduler queue (ray/celery) w/ jobs api
@@ -118,9 +126,9 @@ class MetaReferenceEvalImpl(
         return Job(job_id=job_id, status=JobStatus.completed)
 
     async def _run_model_generation(
-        self, input_rows: list[dict[str, Any]], benchmark_config: BenchmarkConfig
+        self, input_rows: list[dict[str, Any]], request: EvaluateRowsRequest
     ) -> list[dict[str, Any]]:
-        candidate = benchmark_config.eval_candidate
+        candidate = request.benchmark_config.eval_candidate
         assert candidate.sampling_params.max_tokens is not None, "SamplingParams.max_tokens must be provided"
         sampling_params = {"max_tokens": candidate.sampling_params.max_tokens}
 
@@ -165,50 +173,50 @@ class MetaReferenceEvalImpl(
 
     async def evaluate_rows(
         self,
-        benchmark_id: str,
-        input_rows: list[dict[str, Any]],
-        scoring_functions: list[str],
-        benchmark_config: BenchmarkConfig,
+        request: EvaluateRowsRequest,
     ) -> EvaluateResponse:
-        candidate = benchmark_config.eval_candidate
+        candidate = request.benchmark_config.eval_candidate
         # Agent evaluation removed
         if candidate.type == "model":
-            generations = await self._run_model_generation(input_rows, benchmark_config)
+            generations = await self._run_model_generation(request.input_rows, request)
         else:
             raise ValueError(f"Invalid candidate type: {candidate.type}")
 
         # scoring with generated_answer
         score_input_rows = [
-            input_r | generated_r for input_r, generated_r in zip(input_rows, generations, strict=False)
+            input_r | generated_r for input_r, generated_r in zip(request.input_rows, generations, strict=False)
         ]
 
-        if benchmark_config.scoring_params is not None:
+        if request.benchmark_config.scoring_params is not None:
             scoring_functions_dict = {
-                scoring_fn_id: benchmark_config.scoring_params.get(scoring_fn_id, None)
-                for scoring_fn_id in scoring_functions
+                scoring_fn_id: request.benchmark_config.scoring_params.get(scoring_fn_id, None)
+                for scoring_fn_id in request.scoring_functions
             }
         else:
-            scoring_functions_dict = dict.fromkeys(scoring_functions)
+            scoring_functions_dict = dict.fromkeys(request.scoring_functions)
 
-        score_response = await self.scoring_api.score(
-            input_rows=score_input_rows, scoring_functions=scoring_functions_dict
+        score_request = ScoreRequest(
+            input_rows=score_input_rows,
+            scoring_functions=scoring_functions_dict,
         )
+        score_response = await self.scoring_api.score(score_request)
 
         return EvaluateResponse(generations=generations, scores=score_response.results)
 
-    async def job_status(self, benchmark_id: str, job_id: str) -> Job:
-        if job_id in self.jobs:
-            return Job(job_id=job_id, status=JobStatus.completed)
+    async def job_status(self, request: JobStatusRequest) -> Job:
+        if request.job_id in self.jobs:
+            return Job(job_id=request.job_id, status=JobStatus.completed)
 
-        raise ValueError(f"Job {job_id} not found")
+        raise ValueError(f"Job {request.job_id} not found")
 
-    async def job_cancel(self, benchmark_id: str, job_id: str) -> None:
+    async def job_cancel(self, request: JobCancelRequest) -> None:
         raise NotImplementedError("Job cancel is not implemented yet")
 
-    async def job_result(self, benchmark_id: str, job_id: str) -> EvaluateResponse:
-        job = await self.job_status(benchmark_id, job_id)
+    async def job_result(self, request: JobResultRequest) -> EvaluateResponse:
+        job_status_request = JobStatusRequest(benchmark_id=request.benchmark_id, job_id=request.job_id)
+        job = await self.job_status(job_status_request)
         status = job.status
         if not status or status != JobStatus.completed:
             raise ValueError(f"Job is not completed, Status: {status.value}")
 
-        return self.jobs[job_id]
+        return self.jobs[request.job_id]

llama_stack/providers/inline/ios/inference/LocalInferenceImpl/LocalInference.h

@@ -0,0 +1,9 @@
+#import <Foundation/Foundation.h>
+
+//! Project version number for LocalInference.
+FOUNDATION_EXPORT double LocalInferenceVersionNumber;
+
+//! Project version string for LocalInference.
+FOUNDATION_EXPORT const unsigned char LocalInferenceVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <LocalInference/PublicHeader.h>