llama-stack-api 0.4.3__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack_api/eval/compat.py

@@ -0,0 +1,300 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Backward compatibility helpers for the Eval API.
+
+This module provides utilities to support both the old-style (individual parameters)
+and new-style (request objects) calling conventions for Eval API methods.
+
+The old-style parameters are deprecated and will be removed in a future release.
+
+Note: When both a request object AND individual parameters are provided, the request
+object takes precedence and individual parameters are ignored.
+"""
+
+import warnings
+from typing import Any
+
+from .models import (
+    BenchmarkConfig,
+    EvaluateRowsRequest,
+    JobCancelRequest,
+    JobResultRequest,
+    JobStatusRequest,
+    RunEvalRequest,
+)
+
+_DEPRECATION_TARGET = "0.6.0"
+
+_DEPRECATION_MESSAGE = (
+    "Passing individual parameters to {method_name}() is deprecated. "
+    "Please use {request_class}(benchmark_id=..., ...) instead. "
+    "This will be removed in version {target}."
+)
+
+
+def _emit_deprecation_warning(method_name: str, request_class: str) -> None:
+    """Emit a deprecation warning for old-style parameter usage."""
+    warnings.warn(
+        _DEPRECATION_MESSAGE.format(method_name=method_name, request_class=request_class, target=_DEPRECATION_TARGET),
+        DeprecationWarning,
+        stacklevel=4,
+    )
+
+
+def _format_missing_params(required: list[str], provided: dict[str, Any]) -> str:
+    """Format error message showing which parameters are missing."""
+    missing = [p for p in required if provided.get(p) is None]
+    provided_names = [p for p in required if provided.get(p) is not None]
+
+    parts = []
+    if missing:
+        parts.append(f"missing: {', '.join(missing)}")
+    if provided_names:
+        parts.append(f"provided: {', '.join(provided_names)}")
+
+    return "; ".join(parts)
+
+
+def _validate_not_empty(value: Any, name: str) -> None:
+    """Validate that a value is not None, empty string, or empty list."""
+    if not value:
+        raise ValueError(f"'{name}' cannot be None or empty. Provided: {value}")
+
+
+def resolve_run_eval_request(
+    request: RunEvalRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    benchmark_config: BenchmarkConfig | None = None,
+) -> RunEvalRequest:
+    """
+    Resolve run_eval parameters to a RunEvalRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        benchmark_config: (Deprecated) The benchmark configuration
+
+    Returns:
+        RunEvalRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.benchmark_config, "benchmark_config")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and benchmark_config:
+        _emit_deprecation_warning("run_eval", "RunEvalRequest")
+        return RunEvalRequest(
+            benchmark_id=benchmark_id,
+            benchmark_config=benchmark_config,
+        )
+
+    required = ["benchmark_id", "benchmark_config"]
+    provided = {"benchmark_id": benchmark_id, "benchmark_config": benchmark_config}
+    raise ValueError(
+        f"Either 'request' (RunEvalRequest) or both 'benchmark_id' and 'benchmark_config' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_evaluate_rows_request(
+    request: EvaluateRowsRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    input_rows: list[dict[str, Any]] | None = None,
+    scoring_functions: list[str] | None = None,
+    benchmark_config: BenchmarkConfig | None = None,
+) -> EvaluateRowsRequest:
+    """
+    Resolve evaluate_rows parameters to an EvaluateRowsRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        input_rows: (Deprecated) The rows to evaluate
+        scoring_functions: (Deprecated) The scoring functions to use
+        benchmark_config: (Deprecated) The benchmark configuration
+
+    Returns:
+        EvaluateRowsRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.input_rows, "input_rows")
+        _validate_not_empty(request.scoring_functions, "scoring_functions")
+        _validate_not_empty(request.benchmark_config, "benchmark_config")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and input_rows and scoring_functions and benchmark_config:
+        _emit_deprecation_warning("evaluate_rows", "EvaluateRowsRequest")
+        return EvaluateRowsRequest(
+            benchmark_id=benchmark_id,
+            input_rows=input_rows,
+            scoring_functions=scoring_functions,
+            benchmark_config=benchmark_config,
+        )
+
+    required = ["benchmark_id", "input_rows", "scoring_functions", "benchmark_config"]
+    provided = {
+        "benchmark_id": benchmark_id,
+        "input_rows": input_rows,
+        "scoring_functions": scoring_functions,
+        "benchmark_config": benchmark_config,
+    }
+    raise ValueError(
+        f"Either 'request' (EvaluateRowsRequest) or all of 'benchmark_id', 'input_rows', "
+        f"'scoring_functions', and 'benchmark_config' must be provided. "
+        f"{_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_job_status_request(
+    request: JobStatusRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    job_id: str | None = None,
+) -> JobStatusRequest:
+    """
+    Resolve job_status parameters to a JobStatusRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        job_id: (Deprecated) The job ID
+
+    Returns:
+        JobStatusRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.job_id, "job_id")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and job_id:
+        _emit_deprecation_warning("job_status", "JobStatusRequest")
+        return JobStatusRequest(
+            benchmark_id=benchmark_id,
+            job_id=job_id,
+        )
+
+    required = ["benchmark_id", "job_id"]
+    provided = {"benchmark_id": benchmark_id, "job_id": job_id}
+    raise ValueError(
+        f"Either 'request' (JobStatusRequest) or both 'benchmark_id' and 'job_id' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_job_cancel_request(
+    request: JobCancelRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    job_id: str | None = None,
+) -> JobCancelRequest:
+    """
+    Resolve job_cancel parameters to a JobCancelRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        job_id: (Deprecated) The job ID
+
+    Returns:
+        JobCancelRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.job_id, "job_id")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and job_id:
+        _emit_deprecation_warning("job_cancel", "JobCancelRequest")
+        return JobCancelRequest(
+            benchmark_id=benchmark_id,
+            job_id=job_id,
+        )
+
+    required = ["benchmark_id", "job_id"]
+    provided = {"benchmark_id": benchmark_id, "job_id": job_id}
+    raise ValueError(
+        f"Either 'request' (JobCancelRequest) or both 'benchmark_id' and 'job_id' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_job_result_request(
+    request: JobResultRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    job_id: str | None = None,
+) -> JobResultRequest:
+    """
+    Resolve job_result parameters to a JobResultRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        job_id: (Deprecated) The job ID
+
+    Returns:
+        JobResultRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.job_id, "job_id")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and job_id:
+        _emit_deprecation_warning("job_result", "JobResultRequest")
+        return JobResultRequest(
+            benchmark_id=benchmark_id,
+            job_id=job_id,
+        )
+
+    required = ["benchmark_id", "job_id"]
+    provided = {"benchmark_id": benchmark_id, "job_id": job_id}
+    raise ValueError(
+        f"Either 'request' (JobResultRequest) or both 'benchmark_id' and 'job_id' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )

llama_stack_api/eval/fastapi_routes.py

@@ -0,0 +1,126 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+
+from llama_stack_api.common.job_types import Job
+from llama_stack_api.router_utils import create_path_dependency, standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1ALPHA
+
+from .api import Eval
+from .models import (
+    BenchmarkIdRequest,
+    EvaluateResponse,
+    EvaluateRowsBodyRequest,
+    EvaluateRowsRequest,
+    JobCancelRequest,
+    JobResultRequest,
+    JobStatusRequest,
+    RunEvalBodyRequest,
+    RunEvalRequest,
+)
+
+get_benchmark_id_request = create_path_dependency(BenchmarkIdRequest)
+
+
+def create_router(impl: Eval) -> APIRouter:
+    """Create a FastAPI router for the Eval API."""
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1ALPHA}",
+        tags=["Eval"],
+        responses=standard_responses,
+    )
+
+    @router.post(
+        "/eval/benchmarks/{benchmark_id}/jobs",
+        response_model=Job,
+        summary="Run Eval",
+        description="Run an evaluation on a benchmark.",
+        responses={
+            200: {"description": "The job that was created to run the evaluation."},
+        },
+    )
+    async def run_eval(
+        benchmark_id_request: Annotated[BenchmarkIdRequest, Depends(get_benchmark_id_request)],
+        body_request: Annotated[RunEvalBodyRequest, Body(...)],
+    ) -> Job:
+        request = RunEvalRequest(
+            benchmark_id=benchmark_id_request.benchmark_id,
+            benchmark_config=body_request.benchmark_config,
+        )
+        return await impl.run_eval(request)
+
+    @router.post(
+        "/eval/benchmarks/{benchmark_id}/evaluations",
+        response_model=EvaluateResponse,
+        summary="Evaluate Rows",
+        description="Evaluate a list of rows on a benchmark.",
+        responses={
+            200: {"description": "EvaluateResponse object containing generations and scores."},
+        },
+    )
+    async def evaluate_rows(
+        benchmark_id_request: Annotated[BenchmarkIdRequest, Depends(get_benchmark_id_request)],
+        body_request: Annotated[EvaluateRowsBodyRequest, Body(...)],
+    ) -> EvaluateResponse:
+        request = EvaluateRowsRequest(
+            benchmark_id=benchmark_id_request.benchmark_id,
+            input_rows=body_request.input_rows,
+            scoring_functions=body_request.scoring_functions,
+            benchmark_config=body_request.benchmark_config,
+        )
+        return await impl.evaluate_rows(request)
+
+    @router.get(
+        "/eval/benchmarks/{benchmark_id}/jobs/{job_id}",
+        response_model=Job,
+        summary="Job Status",
+        description="Get the status of a job.",
+        responses={
+            200: {"description": "The status of the evaluation job."},
+        },
+    )
+    async def job_status(
+        benchmark_id: str,
+        job_id: str,
+    ) -> Job:
+        request = JobStatusRequest(benchmark_id=benchmark_id, job_id=job_id)
+        return await impl.job_status(request)
+
+    @router.delete(
+        "/eval/benchmarks/{benchmark_id}/jobs/{job_id}",
+        summary="Job Cancel",
+        description="Cancel a job.",
+        responses={
+            200: {"description": "Successful Response"},
+        },
+    )
+    async def job_cancel(
+        benchmark_id: str,
+        job_id: str,
+    ) -> None:
+        request = JobCancelRequest(benchmark_id=benchmark_id, job_id=job_id)
+        return await impl.job_cancel(request)
+
+    @router.get(
+        "/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result",
+        response_model=EvaluateResponse,
+        summary="Job Result",
+        description="Get the result of a job.",
+        responses={
+            200: {"description": "The result of the job."},
+        },
+    )
+    async def job_result(
+        benchmark_id: str,
+        job_id: str,
+    ) -> EvaluateResponse:
+        request = JobResultRequest(benchmark_id=benchmark_id, job_id=job_id)
+        return await impl.job_result(request)
+
+    return router

llama_stack_api/eval/models.py

@@ -0,0 +1,141 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from llama_stack_api.inference import SamplingParams, SystemMessage
+from llama_stack_api.schema_utils import json_schema_type
+from llama_stack_api.scoring import ScoringResult
+from llama_stack_api.scoring_functions import ScoringFnParams
+
+
+@json_schema_type
+class ModelCandidate(BaseModel):
+    """A model candidate for evaluation."""
+
+    type: Literal["model"] = "model"
+    model: str = Field(..., description="The model ID to evaluate", min_length=1)
+    sampling_params: SamplingParams = Field(..., description="The sampling parameters for the model")
+    system_message: SystemMessage | None = Field(
+        None, description="The system message providing instructions or context to the model"
+    )
+
+
+EvalCandidate = ModelCandidate
+
+
+@json_schema_type
+class BenchmarkConfig(BaseModel):
+    """A benchmark configuration for evaluation."""
+
+    eval_candidate: EvalCandidate = Field(..., description="The candidate to evaluate")
+    scoring_params: dict[str, ScoringFnParams] = Field(
+        default_factory=dict,
+        description="Map between scoring function id and parameters for each scoring function you want to run",
+    )
+    num_examples: int | None = Field(
+        None,
+        description="Number of examples to evaluate (useful for testing), if not provided, all examples in the dataset will be evaluated",
+        ge=1,
+    )
+    # we could optinally add any specific dataset config here
+
+
+@json_schema_type
+class EvaluateResponse(BaseModel):
+    """The response from an evaluation."""
+
+    generations: list[dict[str, Any]] = Field(..., description="The generations from the evaluation")
+    scores: dict[str, ScoringResult] = Field(
+        ..., description="The scores from the evaluation. Each key in the dict is a scoring function name"
+    )
+
+
+@json_schema_type
+class BenchmarkIdRequest(BaseModel):
+    """Request model containing benchmark_id path parameter."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark", min_length=1)
+
+
+@json_schema_type
+class RunEvalRequest(BaseModel):
+    """Request model for running an evaluation on a benchmark."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark to run the evaluation on", min_length=1)
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class RunEvalBodyRequest(BaseModel):
+    """Request body model for running an evaluation (without path parameter)."""
+
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class EvaluateRowsRequest(BaseModel):
+    """Request model for evaluating a list of rows on a benchmark."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark to run the evaluation on", min_length=1)
+    input_rows: list[dict[str, Any]] = Field(..., description="The rows to evaluate", min_length=1)
+    scoring_functions: list[str] = Field(
+        ..., description="The scoring functions to use for the evaluation", min_length=1
+    )
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class EvaluateRowsBodyRequest(BaseModel):
+    """Request body model for evaluating rows (without path parameter)."""
+
+    input_rows: list[dict[str, Any]] = Field(..., description="The rows to evaluate", min_length=1)
+    scoring_functions: list[str] = Field(
+        ..., description="The scoring functions to use for the evaluation", min_length=1
+    )
+    benchmark_config: BenchmarkConfig = Field(..., description="The configuration for the benchmark")
+
+
+@json_schema_type
+class JobStatusRequest(BaseModel):
+    """Request model for getting the status of a job."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark associated with the job", min_length=1)
+    job_id: str = Field(..., description="The ID of the job to get the status of", min_length=1)
+
+
+@json_schema_type
+class JobCancelRequest(BaseModel):
+    """Request model for canceling a job."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark associated with the job", min_length=1)
+    job_id: str = Field(..., description="The ID of the job to cancel", min_length=1)
+
+
+@json_schema_type
+class JobResultRequest(BaseModel):
+    """Request model for getting the result of a job."""
+
+    benchmark_id: str = Field(..., description="The ID of the benchmark associated with the job", min_length=1)
+    job_id: str = Field(..., description="The ID of the job to get the result of", min_length=1)
+
+
+__all__ = [
+    "ModelCandidate",
+    "EvalCandidate",
+    "BenchmarkConfig",
+    "EvaluateResponse",
+    "BenchmarkIdRequest",
+    "RunEvalRequest",
+    "RunEvalBodyRequest",
+    "EvaluateRowsRequest",
+    "EvaluateRowsBodyRequest",
+    "JobStatusRequest",
+    "JobCancelRequest",
+    "JobResultRequest",
+]

llama_stack_api/file_processors/__init__.py

@@ -0,0 +1,27 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""File Processors API protocol and models.
+
+This module contains the File Processors protocol definition.
+Pydantic models are defined in llama_stack_api.file_processors.models.
+The FastAPI router is defined in llama_stack_api.file_processors.fastapi_routes.
+"""
+
+# Import fastapi_routes for router factory access
+from . import fastapi_routes
+
+# Import protocol for re-export
+from .api import FileProcessors
+
+# Import models for re-export
+from .models import ProcessFileResponse
+
+__all__ = [
+    "FileProcessors",
+    "ProcessFileResponse",
+    "fastapi_routes",
+]

llama_stack_api/file_processors/api.py

@@ -0,0 +1,64 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any, Protocol, runtime_checkable
+
+from fastapi import UploadFile
+
+from llama_stack_api.vector_io import VectorStoreChunkingStrategy
+
+from .models import ProcessFileResponse
+
+
+@runtime_checkable
+class FileProcessors(Protocol):
+    """
+    File Processor API for converting files into structured, processable content.
+
+    This API provides a flexible interface for processing various file formats
+    (PDFs, documents, images, etc.) into normalized text content that can be used for
+    vector store ingestion, RAG applications, or standalone content extraction.
+
+    The API focuses on parsing and normalization:
+    - Multiple file formats through extensible provider architecture
+    - Multipart form uploads or file ID references
+    - Configurable processing options per provider
+    - Optional chunking using provider's native capabilities
+    - Rich metadata about processing results
+
+    For embedding generation, use the chunks from this API with the separate
+    embedding API to maintain clean separation of concerns.
+
+    Future providers can extend this interface to support additional formats,
+    processing capabilities, and optimization strategies.
+    """
+
+    async def process_file(
+        self,
+        file: UploadFile | None = None,
+        file_id: str | None = None,
+        options: dict[str, Any] | None = None,
+        chunking_strategy: VectorStoreChunkingStrategy | None = None,
+    ) -> ProcessFileResponse:
+        """
+        Process a file into chunks ready for vector database storage.
+
+        This method supports two modes of operation via multipart form request:
+        1. Direct upload: Upload and process a file directly (file parameter)
+        2. File storage: Process files already uploaded to file storage (file_id parameter)
+
+        Exactly one of file or file_id must be provided.
+
+        If no chunking_strategy is provided, the entire file content is returned as a single chunk.
+        If chunking_strategy is provided, the file is split according to the strategy.
+
+        :param file: The uploaded file object containing content and metadata (filename, content_type, etc.). Mutually exclusive with file_id.
+        :param file_id: ID of file already uploaded to file storage. Mutually exclusive with file.
+        :param options: Provider-specific processing options (e.g., OCR settings, output format).
+        :param chunking_strategy: Optional strategy for splitting content into chunks.
+        :returns: ProcessFileResponse with chunks ready for vector database storage.
+        """
+        ...