llama-stack-api 0.4.4__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack_api/datasetio/api.py

@@ -0,0 +1,42 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""DatasetIO API protocol definition.
+
+This module contains the DatasetIO protocol definition.
+Pydantic models are defined in llama_stack_api.datasetio.models.
+The FastAPI router is defined in llama_stack_api.datasetio.fastapi_routes.
+"""
+
+from typing import Protocol, runtime_checkable
+
+from llama_stack_api.datasets import Dataset
+
+from .models import (
+    AppendRowsRequest,
+    IterRowsRequest,
+    PaginatedResponse,
+)
+
+
+class DatasetStore(Protocol):
+    def get_dataset(self, dataset_id: str) -> Dataset: ...
+
+
+@runtime_checkable
+class DatasetIO(Protocol):
+    """Protocol for dataset I/O operations.
+
+    The DatasetIO API provides operations for reading and writing data to datasets.
+    This includes iterating over rows and appending new rows to existing datasets.
+    """
+
+    # keeping for aligning with inference/safety, but this is not used
+    dataset_store: DatasetStore
+
+    async def iterrows(self, request: IterRowsRequest) -> PaginatedResponse: ...
+
+    async def append_rows(self, request: AppendRowsRequest) -> None: ...

llama_stack_api/datasetio/fastapi_routes.py

@@ -0,0 +1,94 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""FastAPI router for the DatasetIO API.
+
+This module defines the FastAPI router for the DatasetIO API using standard
+FastAPI route decorators.
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Path, Query
+
+from llama_stack_api.common.responses import PaginatedResponse
+from llama_stack_api.router_utils import standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1BETA
+
+from .api import DatasetIO
+from .models import (
+    AppendRowsRequest,
+    IterRowsRequest,
+)
+
+
+def create_router(impl: DatasetIO) -> APIRouter:
+    """Create a FastAPI router for the DatasetIO API.
+
+    Args:
+        impl: The DatasetIO implementation instance
+
+    Returns:
+        APIRouter configured for the DatasetIO API
+    """
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1BETA}",
+        tags=["DatasetIO"],
+        responses=standard_responses,
+    )
+
+    @router.get(
+        "/datasetio/iterrows/{dataset_id:path}",
+        response_model=PaginatedResponse,
+        summary="Get a paginated list of rows from a dataset.",
+        description="""Get a paginated list of rows from a dataset.
+
+Uses offset-based pagination where:
+- start_index: The starting index (0-based). If None, starts from beginning.
+- limit: Number of items to return. If None or -1, returns all items.
+
+The response includes:
+- data: List of items for the current page.
+- has_more: Whether there are more items available after this set.""",
+        responses={
+            200: {"description": "A PaginatedResponse containing the rows."},
+        },
+    )
+    async def iterrows(
+        dataset_id: Annotated[str, Path(description="The ID of the dataset to get the rows from.")],
+        start_index: Annotated[
+            int | None, Query(description="Index into dataset for the first row to get. Get all rows if None.")
+        ] = None,
+        limit: Annotated[int | None, Query(description="The number of rows to get.")] = None,
+    ) -> PaginatedResponse:
+        request = IterRowsRequest(
+            dataset_id=dataset_id,
+            start_index=start_index,
+            limit=limit,
+        )
+        return await impl.iterrows(request)
+
+    @router.post(
+        "/datasetio/append-rows/{dataset_id:path}",
+        status_code=204,
+        summary="Append rows to a dataset.",
+        description="Append rows to a dataset.",
+        responses={
+            204: {"description": "Rows were successfully appended."},
+        },
+    )
+    async def append_rows(
+        dataset_id: Annotated[str, Path(description="The ID of the dataset to append the rows to.")],
+        request: Annotated[AppendRowsRequest, Body(...)],
+    ) -> None:
+        # Override the dataset_id from the path
+        request_with_id = AppendRowsRequest(
+            dataset_id=dataset_id,
+            rows=request.rows,
+        )
+        return await impl.append_rows(request_with_id)
+
+    return router

llama_stack_api/datasetio/models.py

@@ -0,0 +1,48 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Pydantic models for DatasetIO API requests and responses.
+
+This module defines the request and response models for the DatasetIO API
+using Pydantic with Field descriptions for OpenAPI schema generation.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from llama_stack_api.common.responses import PaginatedResponse
+from llama_stack_api.schema_utils import json_schema_type
+
+
+@json_schema_type
+class IterRowsRequest(BaseModel):
+    """Request model for iterating over rows in a dataset."""
+
+    dataset_id: str = Field(..., description="The ID of the dataset to get the rows from.")
+    start_index: int | None = Field(
+        default=None,
+        description="Index into dataset for the first row to get. Get all rows if None.",
+    )
+    limit: int | None = Field(
+        default=None,
+        description="The number of rows to get.",
+    )
+
+
+@json_schema_type
+class AppendRowsRequest(BaseModel):
+    """Request model for appending rows to a dataset."""
+
+    dataset_id: str = Field(..., description="The ID of the dataset to append the rows to.")
+    rows: list[dict[str, Any]] = Field(..., description="The rows to append to the dataset.")
+
+
+__all__ = [
+    "AppendRowsRequest",
+    "IterRowsRequest",
+    "PaginatedResponse",
+]

llama_stack_api/eval/__init__.py

@@ -0,0 +1,55 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from llama_stack_api.common.job_types import Job
+
+from . import fastapi_routes
+from .api import Eval
+from .compat import (
+    resolve_evaluate_rows_request,
+    resolve_job_cancel_request,
+    resolve_job_result_request,
+    resolve_job_status_request,
+    resolve_run_eval_request,
+)
+from .models import (
+    BenchmarkConfig,
+    BenchmarkIdRequest,
+    EvalCandidate,
+    EvaluateResponse,
+    EvaluateRowsBodyRequest,
+    EvaluateRowsRequest,
+    JobCancelRequest,
+    JobResultRequest,
+    JobStatusRequest,
+    ModelCandidate,
+    RunEvalBodyRequest,
+    RunEvalRequest,
+)
+
+__all__ = [
+    "Eval",
+    "BenchmarkConfig",
+    "BenchmarkIdRequest",
+    "EvalCandidate",
+    "EvaluateResponse",
+    "EvaluateRowsBodyRequest",
+    "EvaluateRowsRequest",
+    "Job",
+    "JobCancelRequest",
+    "JobResultRequest",
+    "JobStatusRequest",
+    "ModelCandidate",
+    "RunEvalBodyRequest",
+    "RunEvalRequest",
+    "fastapi_routes",
+    # Backward compatibility helpers
+    "resolve_run_eval_request",
+    "resolve_evaluate_rows_request",
+    "resolve_job_status_request",
+    "resolve_job_cancel_request",
+    "resolve_job_result_request",
+]

llama_stack_api/eval/api.py

@@ -0,0 +1,51 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Protocol, runtime_checkable
+
+from llama_stack_api.common.job_types import Job
+
+from .models import (
+    EvaluateResponse,
+    EvaluateRowsRequest,
+    JobCancelRequest,
+    JobResultRequest,
+    JobStatusRequest,
+    RunEvalRequest,
+)
+
+
+@runtime_checkable
+class Eval(Protocol):
+    """Evaluations
+
+    Llama Stack Evaluation API for running evaluations on model and agent candidates."""
+
+    async def run_eval(
+        self,
+        request: RunEvalRequest,
+    ) -> Job:
+        """Run an evaluation on a benchmark."""
+        ...
+
+    async def evaluate_rows(
+        self,
+        request: EvaluateRowsRequest,
+    ) -> EvaluateResponse:
+        """Evaluate a list of rows on a benchmark."""
+        ...
+
+    async def job_status(self, request: JobStatusRequest) -> Job:
+        """Get the status of a job."""
+        ...
+
+    async def job_cancel(self, request: JobCancelRequest) -> None:
+        """Cancel a job."""
+        ...
+
+    async def job_result(self, request: JobResultRequest) -> EvaluateResponse:
+        """Get the result of a job."""
+        ...

llama_stack_api/eval/compat.py

@@ -0,0 +1,300 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Backward compatibility helpers for the Eval API.
+
+This module provides utilities to support both the old-style (individual parameters)
+and new-style (request objects) calling conventions for Eval API methods.
+
+The old-style parameters are deprecated and will be removed in a future release.
+
+Note: When both a request object AND individual parameters are provided, the request
+object takes precedence and individual parameters are ignored.
+"""
+
+import warnings
+from typing import Any
+
+from .models import (
+    BenchmarkConfig,
+    EvaluateRowsRequest,
+    JobCancelRequest,
+    JobResultRequest,
+    JobStatusRequest,
+    RunEvalRequest,
+)
+
+_DEPRECATION_TARGET = "0.6.0"
+
+_DEPRECATION_MESSAGE = (
+    "Passing individual parameters to {method_name}() is deprecated. "
+    "Please use {request_class}(benchmark_id=..., ...) instead. "
+    "This will be removed in version {target}."
+)
+
+
+def _emit_deprecation_warning(method_name: str, request_class: str) -> None:
+    """Emit a deprecation warning for old-style parameter usage."""
+    warnings.warn(
+        _DEPRECATION_MESSAGE.format(method_name=method_name, request_class=request_class, target=_DEPRECATION_TARGET),
+        DeprecationWarning,
+        stacklevel=4,
+    )
+
+
+def _format_missing_params(required: list[str], provided: dict[str, Any]) -> str:
+    """Format error message showing which parameters are missing."""
+    missing = [p for p in required if provided.get(p) is None]
+    provided_names = [p for p in required if provided.get(p) is not None]
+
+    parts = []
+    if missing:
+        parts.append(f"missing: {', '.join(missing)}")
+    if provided_names:
+        parts.append(f"provided: {', '.join(provided_names)}")
+
+    return "; ".join(parts)
+
+
+def _validate_not_empty(value: Any, name: str) -> None:
+    """Validate that a value is not None, empty string, or empty list."""
+    if not value:
+        raise ValueError(f"'{name}' cannot be None or empty. Provided: {value}")
+
+
+def resolve_run_eval_request(
+    request: RunEvalRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    benchmark_config: BenchmarkConfig | None = None,
+) -> RunEvalRequest:
+    """
+    Resolve run_eval parameters to a RunEvalRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        benchmark_config: (Deprecated) The benchmark configuration
+
+    Returns:
+        RunEvalRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.benchmark_config, "benchmark_config")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and benchmark_config:
+        _emit_deprecation_warning("run_eval", "RunEvalRequest")
+        return RunEvalRequest(
+            benchmark_id=benchmark_id,
+            benchmark_config=benchmark_config,
+        )
+
+    required = ["benchmark_id", "benchmark_config"]
+    provided = {"benchmark_id": benchmark_id, "benchmark_config": benchmark_config}
+    raise ValueError(
+        f"Either 'request' (RunEvalRequest) or both 'benchmark_id' and 'benchmark_config' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_evaluate_rows_request(
+    request: EvaluateRowsRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    input_rows: list[dict[str, Any]] | None = None,
+    scoring_functions: list[str] | None = None,
+    benchmark_config: BenchmarkConfig | None = None,
+) -> EvaluateRowsRequest:
+    """
+    Resolve evaluate_rows parameters to an EvaluateRowsRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        input_rows: (Deprecated) The rows to evaluate
+        scoring_functions: (Deprecated) The scoring functions to use
+        benchmark_config: (Deprecated) The benchmark configuration
+
+    Returns:
+        EvaluateRowsRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.input_rows, "input_rows")
+        _validate_not_empty(request.scoring_functions, "scoring_functions")
+        _validate_not_empty(request.benchmark_config, "benchmark_config")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and input_rows and scoring_functions and benchmark_config:
+        _emit_deprecation_warning("evaluate_rows", "EvaluateRowsRequest")
+        return EvaluateRowsRequest(
+            benchmark_id=benchmark_id,
+            input_rows=input_rows,
+            scoring_functions=scoring_functions,
+            benchmark_config=benchmark_config,
+        )
+
+    required = ["benchmark_id", "input_rows", "scoring_functions", "benchmark_config"]
+    provided = {
+        "benchmark_id": benchmark_id,
+        "input_rows": input_rows,
+        "scoring_functions": scoring_functions,
+        "benchmark_config": benchmark_config,
+    }
+    raise ValueError(
+        f"Either 'request' (EvaluateRowsRequest) or all of 'benchmark_id', 'input_rows', "
+        f"'scoring_functions', and 'benchmark_config' must be provided. "
+        f"{_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_job_status_request(
+    request: JobStatusRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    job_id: str | None = None,
+) -> JobStatusRequest:
+    """
+    Resolve job_status parameters to a JobStatusRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        job_id: (Deprecated) The job ID
+
+    Returns:
+        JobStatusRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.job_id, "job_id")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and job_id:
+        _emit_deprecation_warning("job_status", "JobStatusRequest")
+        return JobStatusRequest(
+            benchmark_id=benchmark_id,
+            job_id=job_id,
+        )
+
+    required = ["benchmark_id", "job_id"]
+    provided = {"benchmark_id": benchmark_id, "job_id": job_id}
+    raise ValueError(
+        f"Either 'request' (JobStatusRequest) or both 'benchmark_id' and 'job_id' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_job_cancel_request(
+    request: JobCancelRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    job_id: str | None = None,
+) -> JobCancelRequest:
+    """
+    Resolve job_cancel parameters to a JobCancelRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        job_id: (Deprecated) The job ID
+
+    Returns:
+        JobCancelRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.job_id, "job_id")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and job_id:
+        _emit_deprecation_warning("job_cancel", "JobCancelRequest")
+        return JobCancelRequest(
+            benchmark_id=benchmark_id,
+            job_id=job_id,
+        )
+
+    required = ["benchmark_id", "job_id"]
+    provided = {"benchmark_id": benchmark_id, "job_id": job_id}
+    raise ValueError(
+        f"Either 'request' (JobCancelRequest) or both 'benchmark_id' and 'job_id' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )
+
+
+def resolve_job_result_request(
+    request: JobResultRequest | None = None,
+    *,
+    benchmark_id: str | None = None,
+    job_id: str | None = None,
+) -> JobResultRequest:
+    """
+    Resolve job_result parameters to a JobResultRequest object.
+
+    Supports both new-style (request object) and old-style (individual parameters).
+    Old-style usage emits a DeprecationWarning.
+
+    Note: If both request object and individual parameters are provided, the request
+    object takes precedence and individual parameters are ignored.
+
+    Args:
+        request: The new-style request object (preferred)
+        benchmark_id: (Deprecated) The benchmark ID
+        job_id: (Deprecated) The job ID
+
+    Returns:
+        JobResultRequest object
+    """
+    if request is not None:
+        _validate_not_empty(request.benchmark_id, "benchmark_id")
+        _validate_not_empty(request.job_id, "job_id")
+        return request
+
+    # Old-style parameters
+    if benchmark_id and job_id:
+        _emit_deprecation_warning("job_result", "JobResultRequest")
+        return JobResultRequest(
+            benchmark_id=benchmark_id,
+            job_id=job_id,
+        )
+
+    required = ["benchmark_id", "job_id"]
+    provided = {"benchmark_id": benchmark_id, "job_id": job_id}
+    raise ValueError(
+        f"Either 'request' (JobResultRequest) or both 'benchmark_id' and 'job_id' "
+        f"must be provided. {_format_missing_params(required, provided)}"
+    )