llama-stack-api 0.4.4__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack_api/scoring/models.py

@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Pydantic models for Scoring API requests and responses.
+
+This module defines the request and response models for the Scoring API
+using Pydantic with Field descriptions for OpenAPI schema generation.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from llama_stack_api.schema_utils import json_schema_type
+from llama_stack_api.scoring_functions import ScoringFnParams
+
+# mapping of metric to value
+ScoringResultRow = dict[str, Any]
+
+
+@json_schema_type
+class ScoringResult(BaseModel):
+    """
+    A scoring result for a single row.
+    """
+
+    score_rows: list[ScoringResultRow] = Field(
+        ..., description="The scoring result for each row. Each row is a map of column name to value."
+    )
+    aggregated_results: dict[str, Any] = Field(..., description="Map of metric name to aggregated value")
+
+
+@json_schema_type
+class ScoreBatchResponse(BaseModel):
+    """Response from batch scoring operations on datasets."""
+
+    dataset_id: str | None = Field(default=None, description="(Optional) The identifier of the dataset that was scored")
+    results: dict[str, ScoringResult] = Field(..., description="A map of scoring function name to ScoringResult")
+
+
+@json_schema_type
+class ScoreResponse(BaseModel):
+    """
+    The response from scoring.
+    """
+
+    results: dict[str, ScoringResult] = Field(..., description="A map of scoring function name to ScoringResult.")
+
+
+@json_schema_type
+class ScoreRequest(BaseModel):
+    """Request model for scoring a list of rows."""
+
+    input_rows: list[dict[str, Any]] = Field(..., description="The rows to score.")
+    scoring_functions: dict[str, ScoringFnParams | None] = Field(
+        ..., description="The scoring functions to use for the scoring."
+    )
+
+
+@json_schema_type
+class ScoreBatchRequest(BaseModel):
+    """Request model for scoring a batch of rows from a dataset."""
+
+    dataset_id: str = Field(..., description="The ID of the dataset to score.")
+    scoring_functions: dict[str, ScoringFnParams | None] = Field(
+        ..., description="The scoring functions to use for the scoring."
+    )
+    save_results_dataset: bool = Field(default=False, description="Whether to save the results to a dataset.")
+
+
+__all__ = [
+    "ScoringResult",
+    "ScoringResultRow",
+    "ScoreBatchResponse",
+    "ScoreResponse",
+    "ScoreRequest",
+    "ScoreBatchRequest",
+]

llama_stack_api/scoring_functions/__init__.py

@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""ScoringFunctions API protocol and models.
+
+This module contains the ScoringFunctions protocol definition.
+Pydantic models are defined in llama_stack_api.scoring_functions.models.
+The FastAPI router is defined in llama_stack_api.scoring_functions.fastapi_routes.
+"""
+
+from . import fastapi_routes
+from .api import ScoringFunctions
+from .models import (
+    AggregationFunctionType,
+    BasicScoringFnParams,
+    CommonScoringFnFields,
+    GetScoringFunctionRequest,
+    ListScoringFunctionsRequest,
+    ListScoringFunctionsResponse,
+    LLMAsJudgeScoringFnParams,
+    RegexParserScoringFnParams,
+    RegisterScoringFunctionRequest,
+    ScoringFn,
+    ScoringFnInput,
+    ScoringFnParams,
+    ScoringFnParamsType,
+    UnregisterScoringFunctionRequest,
+)
+
+__all__ = [
+    "ScoringFunctions",
+    "ScoringFn",
+    "ScoringFnInput",
+    "ScoringFnParams",
+    "ScoringFnParamsType",
+    "AggregationFunctionType",
+    "LLMAsJudgeScoringFnParams",
+    "RegexParserScoringFnParams",
+    "BasicScoringFnParams",
+    "CommonScoringFnFields",
+    "ListScoringFunctionsResponse",
+    "ListScoringFunctionsRequest",
+    "GetScoringFunctionRequest",
+    "RegisterScoringFunctionRequest",
+    "UnregisterScoringFunctionRequest",
+    "fastapi_routes",
+]

llama_stack_api/scoring_functions/api.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Protocol, runtime_checkable
+
+from .models import (
+    GetScoringFunctionRequest,
+    ListScoringFunctionsRequest,
+    ListScoringFunctionsResponse,
+    RegisterScoringFunctionRequest,
+    ScoringFn,
+    UnregisterScoringFunctionRequest,
+)
+
+
+@runtime_checkable
+class ScoringFunctions(Protocol):
+    async def list_scoring_functions(
+        self,
+        request: ListScoringFunctionsRequest,
+    ) -> ListScoringFunctionsResponse: ...
+
+    async def get_scoring_function(
+        self,
+        request: GetScoringFunctionRequest,
+    ) -> ScoringFn: ...
+
+    async def register_scoring_function(
+        self,
+        request: RegisterScoringFunctionRequest,
+    ) -> None: ...
+
+    async def unregister_scoring_function(
+        self,
+        request: UnregisterScoringFunctionRequest,
+    ) -> None: ...

llama_stack_api/scoring_functions/fastapi_routes.py

@@ -0,0 +1,108 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""FastAPI router for the ScoringFunctions API.
+
+This module defines the FastAPI router for the ScoringFunctions API using standard
+FastAPI route decorators.
+
+The router is defined in the API package to keep all API-related code together.
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+
+from llama_stack_api.router_utils import create_path_dependency, create_query_dependency, standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1
+
+from .api import ScoringFunctions
+from .models import (
+    GetScoringFunctionRequest,
+    ListScoringFunctionsRequest,
+    ListScoringFunctionsResponse,
+    RegisterScoringFunctionRequest,
+    ScoringFn,
+    UnregisterScoringFunctionRequest,
+)
+
+get_list_scoring_functions_request = create_query_dependency(ListScoringFunctionsRequest)
+get_get_scoring_function_request = create_path_dependency(GetScoringFunctionRequest)
+get_unregister_scoring_function_request = create_path_dependency(UnregisterScoringFunctionRequest)
+
+
+def create_router(impl: ScoringFunctions) -> APIRouter:
+    """Create a FastAPI router for the ScoringFunctions API.
+
+    Args:
+        impl: The ScoringFunctions implementation instance
+
+    Returns:
+        APIRouter configured for the ScoringFunctions API
+    """
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1}",
+        tags=["Scoring Functions"],
+        responses=standard_responses,
+    )
+
+    @router.get(
+        "/scoring-functions",
+        response_model=ListScoringFunctionsResponse,
+        summary="List all scoring functions.",
+        description="List all scoring functions.",
+        responses={
+            200: {"description": "A ListScoringFunctionsResponse."},
+        },
+    )
+    async def list_scoring_functions(
+        request: Annotated[ListScoringFunctionsRequest, Depends(get_list_scoring_functions_request)],
+    ) -> ListScoringFunctionsResponse:
+        return await impl.list_scoring_functions(request)
+
+    @router.get(
+        "/scoring-functions/{scoring_fn_id:path}",
+        response_model=ScoringFn,
+        summary="Get a scoring function by its ID.",
+        description="Get a scoring function by its ID.",
+        responses={
+            200: {"description": "A ScoringFn."},
+        },
+    )
+    async def get_scoring_function(
+        request: Annotated[GetScoringFunctionRequest, Depends(get_get_scoring_function_request)],
+    ) -> ScoringFn:
+        return await impl.get_scoring_function(request)
+
+    @router.post(
+        "/scoring-functions",
+        summary="Register a scoring function.",
+        description="Register a scoring function.",
+        responses={
+            200: {"description": "The scoring function was successfully registered."},
+        },
+        deprecated=True,
+    )
+    async def register_scoring_function(
+        request: Annotated[RegisterScoringFunctionRequest, Body(...)],
+    ) -> None:
+        return await impl.register_scoring_function(request)
+
+    @router.delete(
+        "/scoring-functions/{scoring_fn_id:path}",
+        summary="Unregister a scoring function.",
+        description="Unregister a scoring function.",
+        responses={
+            200: {"description": "The scoring function was successfully unregistered."},
+        },
+        deprecated=True,
+    )
+    async def unregister_scoring_function(
+        request: Annotated[UnregisterScoringFunctionRequest, Depends(get_unregister_scoring_function_request)],
+    ) -> None:
+        return await impl.unregister_scoring_function(request)
+
+    return router

llama_stack_api/{scoring_functions.py → scoring_functions/models.py}

@@ -4,26 +4,22 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-# TODO: use enum.StrEnum when we drop support for python 3.10
+"""Pydantic models for ScoringFunctions API requests and responses.
+
+This module defines the request and response models for the ScoringFunctions API
+using Pydantic with Field descriptions for OpenAPI schema generation.
+"""
+
 from enum import StrEnum
-from typing import (
-    Annotated,
-    Any,
-    Literal,
-    Protocol,
-    runtime_checkable,
-)
+from typing import Annotated, Any, Literal
 
 from pydantic import BaseModel, Field
 
 from llama_stack_api.common.type_system import ParamType
 from llama_stack_api.resource import Resource, ResourceType
-from llama_stack_api.schema_utils import json_schema_type, register_schema, webmethod
-from llama_stack_api.version import LLAMA_STACK_API_V1
+from llama_stack_api.schema_utils import json_schema_type, register_schema
 
 
-# Perhaps more structure can be imposed on these functions. Maybe they could be associated
-# with standard metrics so they can be rolled up?
 @json_schema_type
 class ScoringFnParamsType(StrEnum):
     """Types of scoring function parameter configurations.
@@ -117,6 +113,44 @@ ScoringFnParams = Annotated[
 register_schema(ScoringFnParams, name="ScoringFnParams")
 
 
+@json_schema_type
+class ListScoringFunctionsRequest(BaseModel):
+    """Request model for listing scoring functions."""
+
+    pass
+
+
+@json_schema_type
+class GetScoringFunctionRequest(BaseModel):
+    """Request model for getting a scoring function."""
+
+    scoring_fn_id: str = Field(..., description="The ID of the scoring function to get.")
+
+
+@json_schema_type
+class RegisterScoringFunctionRequest(BaseModel):
+    """Request model for registering a scoring function."""
+
+    scoring_fn_id: str = Field(..., description="The ID of the scoring function to register.")
+    description: str = Field(..., description="The description of the scoring function.")
+    return_type: ParamType = Field(..., description="The return type of the scoring function.")
+    provider_scoring_fn_id: str | None = Field(
+        default=None, description="The ID of the provider scoring function to use for the scoring function."
+    )
+    provider_id: str | None = Field(default=None, description="The ID of the provider to use for the scoring function.")
+    params: ScoringFnParams | None = Field(
+        default=None,
+        description="The parameters for the scoring function for benchmark eval, these can be overridden for app eval.",
+    )
+
+
+@json_schema_type
+class UnregisterScoringFunctionRequest(BaseModel):
+    """Request model for unregistering a scoring function."""
+
+    scoring_fn_id: str = Field(..., description="The ID of the scoring function to unregister.")
+
+
 class CommonScoringFnFields(BaseModel):
     description: str | None = None
     metadata: dict[str, Any] = Field(
@@ -157,55 +191,24 @@ class ScoringFnInput(CommonScoringFnFields, BaseModel):
 
 @json_schema_type
 class ListScoringFunctionsResponse(BaseModel):
-    data: list[ScoringFn]
-
-
-@runtime_checkable
-class ScoringFunctions(Protocol):
-    @webmethod(route="/scoring-functions", method="GET", level=LLAMA_STACK_API_V1)
-    async def list_scoring_functions(self) -> ListScoringFunctionsResponse:
-        """List all scoring functions.
-
-        :returns: A ListScoringFunctionsResponse.
-        """
-        ...
-
-    @webmethod(route="/scoring-functions/{scoring_fn_id:path}", method="GET", level=LLAMA_STACK_API_V1)
-    async def get_scoring_function(self, scoring_fn_id: str, /) -> ScoringFn:
-        """Get a scoring function by its ID.
-
-        :param scoring_fn_id: The ID of the scoring function to get.
-        :returns: A ScoringFn.
-        """
-        ...
-
-    @webmethod(route="/scoring-functions", method="POST", level=LLAMA_STACK_API_V1, deprecated=True)
-    async def register_scoring_function(
-        self,
-        scoring_fn_id: str,
-        description: str,
-        return_type: ParamType,
-        provider_scoring_fn_id: str | None = None,
-        provider_id: str | None = None,
-        params: ScoringFnParams | None = None,
-    ) -> None:
-        """Register a scoring function.
-
-        :param scoring_fn_id: The ID of the scoring function to register.
-        :param description: The description of the scoring function.
-        :param return_type: The return type of the scoring function.
-        :param provider_scoring_fn_id: The ID of the provider scoring function to use for the scoring function.
-        :param provider_id: The ID of the provider to use for the scoring function.
-        :param params: The parameters for the scoring function for benchmark eval, these can be overridden for app eval.
-        """
-        ...
-
-    @webmethod(
-        route="/scoring-functions/{scoring_fn_id:path}", method="DELETE", level=LLAMA_STACK_API_V1, deprecated=True
-    )
-    async def unregister_scoring_function(self, scoring_fn_id: str) -> None:
-        """Unregister a scoring function.
-
-        :param scoring_fn_id: The ID of the scoring function to unregister.
-        """
-        ...
+    """Response containing a list of scoring function objects."""
+
+    data: list[ScoringFn] = Field(..., description="List of scoring function objects.")
+
+
+__all__ = [
+    "ScoringFnParamsType",
+    "AggregationFunctionType",
+    "LLMAsJudgeScoringFnParams",
+    "RegexParserScoringFnParams",
+    "BasicScoringFnParams",
+    "ScoringFnParams",
+    "ListScoringFunctionsRequest",
+    "GetScoringFunctionRequest",
+    "RegisterScoringFunctionRequest",
+    "UnregisterScoringFunctionRequest",
+    "CommonScoringFnFields",
+    "ScoringFn",
+    "ScoringFnInput",
+    "ListScoringFunctionsResponse",
+]

llama_stack_api/shields/__init__.py

@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Shields API protocol and models.
+
+This module contains the Shields protocol definition.
+Pydantic models are defined in llama_stack_api.shields.models.
+The FastAPI router is defined in llama_stack_api.shields.fastapi_routes.
+"""
+
+# Import fastapi_routes for router factory access
+from . import fastapi_routes
+
+# Import protocol for re-export
+from .api import Shields
+
+# Import models for re-export
+from .models import (
+    CommonShieldFields,
+    GetShieldRequest,
+    ListShieldsResponse,
+    RegisterShieldRequest,
+    Shield,
+    ShieldInput,
+    UnregisterShieldRequest,
+)
+
+__all__ = [
+    "Shields",
+    "Shield",
+    "ShieldInput",
+    "CommonShieldFields",
+    "ListShieldsResponse",
+    "GetShieldRequest",
+    "RegisterShieldRequest",
+    "UnregisterShieldRequest",
+    "fastapi_routes",
+]

llama_stack_api/shields/api.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Shields API protocol definition.
+
+This module contains the Shields protocol for managing shield resources.
+"""
+
+from typing import Protocol, runtime_checkable
+
+from .models import (
+    GetShieldRequest,
+    ListShieldsResponse,
+    RegisterShieldRequest,
+    Shield,
+    UnregisterShieldRequest,
+)
+
+
+@runtime_checkable
+class Shields(Protocol):
+    async def list_shields(self) -> ListShieldsResponse:
+        """List all shields."""
+        ...
+
+    async def get_shield(self, request: GetShieldRequest) -> Shield:
+        """Get a shield by its identifier."""
+        ...
+
+    async def register_shield(self, request: RegisterShieldRequest) -> Shield:
+        """Register a shield."""
+        ...
+
+    async def unregister_shield(self, request: UnregisterShieldRequest) -> None:
+        """Unregister a shield."""
+        ...

llama_stack_api/shields/fastapi_routes.py

@@ -0,0 +1,104 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""FastAPI router for the Shields API.
+
+This module defines the FastAPI router for the Shields API using standard
+FastAPI route decorators.
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+
+from llama_stack_api.router_utils import create_path_dependency, standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1
+
+from .api import Shields
+from .models import (
+    GetShieldRequest,
+    ListShieldsResponse,
+    RegisterShieldRequest,
+    Shield,
+    UnregisterShieldRequest,
+)
+
+# Automatically generate dependency functions from Pydantic models
+get_get_shield_request = create_path_dependency(GetShieldRequest)
+get_unregister_shield_request = create_path_dependency(UnregisterShieldRequest)
+
+
+def create_router(impl: Shields) -> APIRouter:
+    """Create a FastAPI router for the Shields API.
+
+    Args:
+        impl: The Shields implementation instance
+
+    Returns:
+        APIRouter configured for the Shields API
+    """
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1}",
+        tags=["Shields"],
+        responses=standard_responses,
+    )
+
+    @router.get(
+        "/shields",
+        response_model=ListShieldsResponse,
+        summary="List all shields.",
+        description="List all shields.",
+        responses={
+            200: {"description": "A ListShieldsResponse."},
+        },
+    )
+    async def list_shields() -> ListShieldsResponse:
+        return await impl.list_shields()
+
+    @router.get(
+        "/shields/{identifier:path}",
+        response_model=Shield,
+        summary="Get a shield by its identifier.",
+        description="Get a shield by its identifier.",
+        responses={
+            200: {"description": "A Shield."},
+        },
+    )
+    async def get_shield(
+        request: Annotated[GetShieldRequest, Depends(get_get_shield_request)],
+    ) -> Shield:
+        return await impl.get_shield(request)
+
+    @router.post(
+        "/shields",
+        response_model=Shield,
+        summary="Register a shield.",
+        description="Register a shield.",
+        responses={
+            200: {"description": "A Shield."},
+        },
+        deprecated=True,
+    )
+    async def register_shield(
+        request: Annotated[RegisterShieldRequest, Body(...)],
+    ) -> Shield:
+        return await impl.register_shield(request)
+
+    @router.delete(
+        "/shields/{identifier:path}",
+        summary="Unregister a shield.",
+        description="Unregister a shield.",
+        responses={
+            200: {"description": "The shield was successfully unregistered."},
+        },
+        deprecated=True,
+    )
+    async def unregister_shield(
+        request: Annotated[UnregisterShieldRequest, Depends(get_unregister_shield_request)],
+    ) -> None:
+        return await impl.unregister_shield(request)
+
+    return router