llama-stack-api 0.4.4__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack_api/eval/fastapi_routes.py

@@ -0,0 +1,126 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+
+from llama_stack_api.common.job_types import Job
+from llama_stack_api.router_utils import create_path_dependency, standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1ALPHA
+
+from .api import Eval
+from .models import (
+    BenchmarkIdRequest,
+    EvaluateResponse,
+    EvaluateRowsBodyRequest,
+    EvaluateRowsRequest,
+    JobCancelRequest,
+    JobResultRequest,
+    JobStatusRequest,
+    RunEvalBodyRequest,
+    RunEvalRequest,
+)
+
+get_benchmark_id_request = create_path_dependency(BenchmarkIdRequest)
+
+
+def create_router(impl: Eval) -> APIRouter:
+    """Create a FastAPI router for the Eval API."""
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1ALPHA}",
+        tags=["Eval"],
+        responses=standard_responses,
+    )
+
+    @router.post(
+        "/eval/benchmarks/{benchmark_id}/jobs",
+        response_model=Job,
+        summary="Run Eval",
+        description="Run an evaluation on a benchmark.",
+        responses={
+            200: {"description": "The job that was created to run the evaluation."},
+        },
+    )
+    async def run_eval(
+        benchmark_id_request: Annotated[BenchmarkIdRequest, Depends(get_benchmark_id_request)],
+        body_request: Annotated[RunEvalBodyRequest, Body(...)],
+    ) -> Job:
+        request = RunEvalRequest(
+            benchmark_id=benchmark_id_request.benchmark_id,
+            benchmark_config=body_request.benchmark_config,
+        )
+        return await impl.run_eval(request)
+
+    @router.post(
+        "/eval/benchmarks/{benchmark_id}/evaluations",
+        response_model=EvaluateResponse,
+        summary="Evaluate Rows",
+        description="Evaluate a list of rows on a benchmark.",
+        responses={
+            200: {"description": "EvaluateResponse object containing generations and scores."},
+        },
+    )
+    async def evaluate_rows(
+        benchmark_id_request: Annotated[BenchmarkIdRequest, Depends(get_benchmark_id_request)],
+        body_request: Annotated[EvaluateRowsBodyRequest, Body(...)],
+    ) -> EvaluateResponse:
+        request = EvaluateRowsRequest(
+            benchmark_id=benchmark_id_request.benchmark_id,
+            input_rows=body_request.input_rows,
+            scoring_functions=body_request.scoring_functions,
+            benchmark_config=body_request.benchmark_config,
+        )
+        return await impl.evaluate_rows(request)
+
+    @router.get(
+        "/eval/benchmarks/{benchmark_id}/jobs/{job_id}",
+        response_model=Job,
+        summary="Job Status",
+        description="Get the status of a job.",
+        responses={
+            200: {"description": "The status of the evaluation job."},
+        },
+    )
+    async def job_status(
+        benchmark_id: str,
+        job_id: str,
+    ) -> Job:
+        request = JobStatusRequest(benchmark_id=benchmark_id, job_id=job_id)
+        return await impl.job_status(request)
+
+    @router.delete(
+        "/eval/benchmarks/{benchmark_id}/jobs/{job_id}",
+        summary="Job Cancel",
+        description="Cancel a job.",
+        responses={
+            200: {"description": "Successful Response"},
+        },
+    )
+    async def job_cancel(
+        benchmark_id: str,
+        job_id: str,
+    ) -> None:
+        request = JobCancelRequest(benchmark_id=benchmark_id, job_id=job_id)
+        return await impl.job_cancel(request)
+
+    @router.get(
+        "/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result",
+        response_model=EvaluateResponse,
+        summary="Job Result",
+        description="Get the result of a job.",
+        responses={
+            200: {"description": "The result of the job."},
+        },
+    )
+    async def job_result(
+        benchmark_id: str,
+        job_id: str,
+    ) -> EvaluateResponse:
+        request = JobResultRequest(benchmark_id=benchmark_id, job_id=job_id)
+        return await impl.job_result(request)
+
+    return router

llama_stack_api/eval/models.py

@@ -0,0 +1,141 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from llama_stack_api.inference import SamplingParams, SystemMessage
+from llama_stack_api.schema_utils import json_schema_type
+from llama_stack_api.scoring import ScoringResult
+from llama_stack_api.scoring_functions import ScoringFnParams
+
+
+@json_schema_type
+class ModelCandidate(BaseModel):
+    """A model candidate for evaluation."""
+
+    type: Literal["model"] = "model"
+    model: str = Field(..., description="The model ID to evaluate", min_length=1)
+    sampling_params: SamplingParams = Field(..., description="The sampling parameters for the model")
+    system_message: SystemMessage | None = Field(
+        None, description="The system message providing instructions or context to the model"
+    )
+
+
+EvalCandidate = ModelCandidate
+
+
+@json_schema_type
+class BenchmarkConfig(BaseModel):
+    """A benchmark configuration for evaluation."""
+
+    eval_candidate: EvalCandidate = Field(..., description="The candidate to evaluate")
+    scoring_params: dict[str, ScoringFnParams] = Field(
+        default_factory=dict,
+        description="Map between scoring function id and parameters for each scoring function you want to run",
+    )
+    num_examples: int | None = Field(
+        None,
+        description="Number of examples to evaluate (useful for testing), if not provided, all examples in the dataset will be evaluated",
+        ge=1,
+    )
+    # we could optinally add any specific dataset config here
+
+
+@json_schema_type
+class EvaluateResponse(BaseModel):
+    """The response from an evaluation."""
+
+    generations: list[dict[str, Any]] = Field(..., description="The generations from the evaluation")
+    scores: dict[str, ScoringResult] = Field(
+        ..., description="The scores from the evaluation. Each key in the dict is a scoring function name"
+    )
+
+
+@json_schema_type
+class BenchmarkIdRequest(BaseModel):
+    """Request model containing benchmark_id path parameter."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark", min_length=1)
+
+
+@json_schema_type
+class RunEvalRequest(BaseModel):
+    """Request model for running an evaluation on a benchmark."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark to run the evaluation on", min_length=1)
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class RunEvalBodyRequest(BaseModel):
+    """Request body model for running an evaluation (without path parameter)."""
+
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class EvaluateRowsRequest(BaseModel):
+    """Request model for evaluating a list of rows on a benchmark."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark to run the evaluation on", min_length=1)
+    input_rows: list[dict[str, Any]] = Field(..., description="The rows to evaluate", min_length=1)
+    scoring_functions: list[str] = Field(
+        ..., description="The scoring functions to use for the evaluation", min_length=1
+    )
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class EvaluateRowsBodyRequest(BaseModel):
+    """Request body model for evaluating rows (without path parameter)."""
+
+    input_rows: list[dict[str, Any]] = Field(..., description="The rows to evaluate", min_length=1)
+    scoring_functions: list[str] = Field(
+        ..., description="The scoring functions to use for the evaluation", min_length=1
+    )
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class JobStatusRequest(BaseModel):
+    """Request model for getting the status of a job."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark associated with the job", min_length=1)
+    job_id: str = Field(..., description="The ID of the job to get the status of", min_length=1)
+
+
+@json_schema_type
+class JobCancelRequest(BaseModel):
+    """Request model for canceling a job."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark associated with the job", min_length=1)
+    job_id: str = Field(..., description="The ID of the job to cancel", min_length=1)
+
+
+@json_schema_type
+class JobResultRequest(BaseModel):
+    """Request model for getting the result of a job."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark associated with the job", min_length=1)
+    job_id: str = Field(..., description="The ID of the job to get the result of", min_length=1)
+
+
+__all__ = [
+    "ModelCandidate",
+    "EvalCandidate",
+    "BenchmarkConfig",
+    "EvaluateResponse",
+    "BenchmarkIdRequest",
+    "RunEvalRequest",
+    "RunEvalBodyRequest",
+    "EvaluateRowsRequest",
+    "EvaluateRowsBodyRequest",
+    "JobStatusRequest",
+    "JobCancelRequest",
+    "JobResultRequest",
+]

llama_stack_api/inference/__init__.py

@@ -0,0 +1,207 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Inference API protocol and models.
+
+This module contains the Inference protocol definition.
+Pydantic models are defined in llama_stack_api.inference.models.
+The FastAPI router is defined in llama_stack_api.inference.fastapi_routes.
+"""
+
+# Import common types for backward compatibility
+# (these were previously available from the old inference.py)
+from llama_stack_api.common.content_types import InterleavedContent
+
+# Import fastapi_routes for router factory access
+from . import fastapi_routes
+
+# Import protocol for re-export
+from .api import Inference, InferenceProvider, ModelStore
+
+# Import models for re-export
+from .models import (
+    AllowedToolsConfig,
+    Bf16QuantizationConfig,
+    ChatCompletionResponseEventType,
+    CompletionRequest,
+    CustomToolConfig,
+    EmbeddingsResponse,
+    EmbeddingTaskType,
+    Fp8QuantizationConfig,
+    FunctionToolConfig,
+    GetChatCompletionRequest,
+    GrammarResponseFormat,
+    GreedySamplingStrategy,
+    Int4QuantizationConfig,
+    JsonSchemaResponseFormat,
+    ListChatCompletionsRequest,
+    ListOpenAIChatCompletionResponse,
+    LogProbConfig,
+    OpenAIAssistantMessageParam,
+    OpenAIChatCompletion,
+    OpenAIChatCompletionChunk,
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartParam,
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionMessageContent,
+    OpenAIChatCompletionRequestWithExtraBody,
+    OpenAIChatCompletionTextOnlyMessageContent,
+    OpenAIChatCompletionToolCall,
+    OpenAIChatCompletionToolCallFunction,
+    OpenAIChatCompletionToolChoice,
+    OpenAIChatCompletionToolChoiceAllowedTools,
+    OpenAIChatCompletionToolChoiceCustomTool,
+    OpenAIChatCompletionToolChoiceFunctionTool,
+    OpenAIChatCompletionUsage,
+    OpenAIChatCompletionUsageCompletionTokensDetails,
+    OpenAIChatCompletionUsagePromptTokensDetails,
+    OpenAIChoice,
+    OpenAIChoiceDelta,
+    OpenAIChoiceLogprobs,
+    OpenAIChunkChoice,
+    OpenAICompletion,
+    OpenAICompletionChoice,
+    OpenAICompletionLogprobs,
+    OpenAICompletionRequestWithExtraBody,
+    OpenAICompletionWithInputMessages,
+    OpenAIDeveloperMessageParam,
+    OpenAIEmbeddingData,
+    OpenAIEmbeddingsRequestWithExtraBody,
+    OpenAIEmbeddingsResponse,
+    OpenAIEmbeddingUsage,
+    OpenAIFile,
+    OpenAIFileFile,
+    OpenAIFinishReason,
+    OpenAIImageURL,
+    OpenAIJSONSchema,
+    OpenAIMessageParam,
+    OpenAIResponseFormatJSONObject,
+    OpenAIResponseFormatJSONSchema,
+    OpenAIResponseFormatParam,
+    OpenAIResponseFormatText,
+    OpenAISystemMessageParam,
+    OpenAITokenLogProb,
+    OpenAIToolMessageParam,
+    OpenAITopLogProb,
+    OpenAIUserMessageParam,
+    QuantizationConfig,
+    QuantizationType,
+    RerankData,
+    RerankRequest,
+    RerankResponse,
+    ResponseFormat,
+    ResponseFormatType,
+    SamplingParams,
+    SamplingStrategy,
+    SystemMessage,
+    SystemMessageBehavior,
+    TextTruncation,
+    TokenLogProbs,
+    ToolChoice,
+    ToolResponseMessage,
+    TopKSamplingStrategy,
+    TopPSamplingStrategy,
+    UserMessage,
+)
+
+__all__ = [
+    # Protocol
+    "Inference",
+    "InferenceProvider",
+    "ModelStore",
+    # Common types (for backward compatibility)
+    "InterleavedContent",
+    # Sampling
+    "GreedySamplingStrategy",
+    "TopPSamplingStrategy",
+    "TopKSamplingStrategy",
+    "SamplingStrategy",
+    "SamplingParams",
+    "LogProbConfig",
+    # Quantization
+    "QuantizationType",
+    "Fp8QuantizationConfig",
+    "Bf16QuantizationConfig",
+    "Int4QuantizationConfig",
+    "QuantizationConfig",
+    # Messages
+    "UserMessage",
+    "SystemMessage",
+    "ToolResponseMessage",
+    "ToolChoice",
+    "TokenLogProbs",
+    # Response
+    "ChatCompletionResponseEventType",
+    "ResponseFormatType",
+    "JsonSchemaResponseFormat",
+    "GrammarResponseFormat",
+    "ResponseFormat",
+    "CompletionRequest",
+    "SystemMessageBehavior",
+    "EmbeddingsResponse",
+    "RerankData",
+    "RerankResponse",
+    # OpenAI Compatibility
+    "OpenAIChatCompletionContentPartTextParam",
+    "OpenAIImageURL",
+    "OpenAIChatCompletionContentPartImageParam",
+    "OpenAIFileFile",
+    "OpenAIFile",
+    "OpenAIChatCompletionContentPartParam",
+    "OpenAIChatCompletionMessageContent",
+    "OpenAIChatCompletionTextOnlyMessageContent",
+    "OpenAIUserMessageParam",
+    "OpenAISystemMessageParam",
+    "OpenAIChatCompletionToolCallFunction",
+    "OpenAIChatCompletionToolCall",
+    "OpenAIAssistantMessageParam",
+    "OpenAIToolMessageParam",
+    "OpenAIDeveloperMessageParam",
+    "OpenAIMessageParam",
+    "OpenAIResponseFormatText",
+    "OpenAIJSONSchema",
+    "OpenAIResponseFormatJSONSchema",
+    "OpenAIResponseFormatJSONObject",
+    "OpenAIResponseFormatParam",
+    "FunctionToolConfig",
+    "OpenAIChatCompletionToolChoiceFunctionTool",
+    "CustomToolConfig",
+    "OpenAIChatCompletionToolChoiceCustomTool",
+    "AllowedToolsConfig",
+    "OpenAIChatCompletionToolChoiceAllowedTools",
+    "OpenAIChatCompletionToolChoice",
+    "OpenAITopLogProb",
+    "OpenAITokenLogProb",
+    "OpenAIChoiceLogprobs",
+    "OpenAIChoiceDelta",
+    "OpenAIChunkChoice",
+    "OpenAIChoice",
+    "OpenAIChatCompletionUsageCompletionTokensDetails",
+    "OpenAIChatCompletionUsagePromptTokensDetails",
+    "OpenAIChatCompletionUsage",
+    "OpenAIChatCompletion",
+    "OpenAIChatCompletionChunk",
+    "OpenAICompletionLogprobs",
+    "OpenAICompletionChoice",
+    "OpenAICompletion",
+    "OpenAIFinishReason",
+    "OpenAIEmbeddingData",
+    "OpenAIEmbeddingUsage",
+    "OpenAIEmbeddingsResponse",
+    "TextTruncation",
+    "EmbeddingTaskType",
+    "OpenAICompletionWithInputMessages",
+    "ListOpenAIChatCompletionResponse",
+    "OpenAICompletionRequestWithExtraBody",
+    "OpenAIChatCompletionRequestWithExtraBody",
+    "OpenAIEmbeddingsRequestWithExtraBody",
+    # Request Models
+    "ListChatCompletionsRequest",
+    "GetChatCompletionRequest",
+    "RerankRequest",
+    # Router factory module
+    "fastapi_routes",
+]

llama_stack_api/inference/api.py

@@ -0,0 +1,93 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from collections.abc import AsyncIterator
+from typing import Protocol, runtime_checkable
+
+from llama_stack_api.models import Model
+
+from .models import (
+    GetChatCompletionRequest,
+    ListChatCompletionsRequest,
+    ListOpenAIChatCompletionResponse,
+    OpenAIChatCompletion,
+    OpenAIChatCompletionChunk,
+    OpenAIChatCompletionRequestWithExtraBody,
+    OpenAICompletion,
+    OpenAICompletionRequestWithExtraBody,
+    OpenAICompletionWithInputMessages,
+    OpenAIEmbeddingsRequestWithExtraBody,
+    OpenAIEmbeddingsResponse,
+    RerankRequest,
+    RerankResponse,
+)
+
+
+class ModelStore(Protocol):
+    async def get_model(self, identifier: str) -> Model: ...
+
+
+@runtime_checkable
+class InferenceProvider(Protocol):
+    """
+    This protocol defines the interface that should be implemented by all inference providers.
+    """
+
+    API_NAMESPACE: str = "Inference"
+
+    model_store: ModelStore | None = None
+
+    async def rerank(
+        self,
+        request: RerankRequest,
+    ) -> RerankResponse:
+        """Rerank a list of documents based on their relevance to a query."""
+        raise NotImplementedError("Reranking is not implemented")
+        return  # this is so mypy's safe-super rule will consider the method concrete
+
+    async def openai_completion(
+        self,
+        params: OpenAICompletionRequestWithExtraBody,
+    ) -> OpenAICompletion | AsyncIterator[OpenAICompletion]:
+        """Generate an OpenAI-compatible completion for the given prompt using the specified model."""
+        ...
+
+    async def openai_chat_completion(
+        self,
+        params: OpenAIChatCompletionRequestWithExtraBody,
+    ) -> OpenAIChatCompletion | AsyncIterator[OpenAIChatCompletionChunk]:
+        """Generate an OpenAI-compatible chat completion for the given messages using the specified model."""
+        ...
+
+    async def openai_embeddings(
+        self,
+        params: OpenAIEmbeddingsRequestWithExtraBody,
+    ) -> OpenAIEmbeddingsResponse:
+        """Generate OpenAI-compatible embeddings for the given input using the specified model."""
+        ...
+
+
+class Inference(InferenceProvider):
+    """Inference
+
+    Llama Stack Inference API for generating completions, chat completions, and embeddings.
+
+    This API provides the raw interface to the underlying models. Three kinds of models are supported:
+    - LLM models: these models generate "raw" and "chat" (conversational) completions.
+    - Embedding models: these models generate embeddings to be used for semantic search.
+    - Rerank models: these models reorder the documents based on their relevance to a query.
+    """
+
+    async def list_chat_completions(
+        self,
+        request: ListChatCompletionsRequest,
+    ) -> ListOpenAIChatCompletionResponse:
+        """List stored chat completions."""
+        raise NotImplementedError("List chat completions is not implemented")
+
+    async def get_chat_completion(self, request: GetChatCompletionRequest) -> OpenAICompletionWithInputMessages:
+        """Retrieve a stored chat completion by its ID."""
+        raise NotImplementedError("Get chat completion is not implemented")