llama-stack-api 0.4.3__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack_api/scoring/__init__.py

@@ -0,0 +1,66 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Scoring API protocol and models.
+
+This module contains the Scoring protocol definition.
+Pydantic models are defined in llama_stack_api.scoring.models.
+The FastAPI router is defined in llama_stack_api.scoring.fastapi_routes.
+"""
+
+# Import fastapi_routes for router factory access
+# Import scoring_functions for re-export
+from llama_stack_api.scoring_functions import (
+    AggregationFunctionType,
+    BasicScoringFnParams,
+    CommonScoringFnFields,
+    ListScoringFunctionsResponse,
+    LLMAsJudgeScoringFnParams,
+    RegexParserScoringFnParams,
+    ScoringFn,
+    ScoringFnInput,
+    ScoringFnParams,
+    ScoringFnParamsType,
+    ScoringFunctions,
+)
+
+from . import fastapi_routes
+
+# Import protocol for FastAPI router
+from .api import Scoring, ScoringFunctionStore
+
+# Import models for re-export
+from .models import (
+    ScoreBatchRequest,
+    ScoreBatchResponse,
+    ScoreRequest,
+    ScoreResponse,
+    ScoringResult,
+    ScoringResultRow,
+)
+
+__all__ = [
+    "Scoring",
+    "ScoringFunctionStore",
+    "ScoringResult",
+    "ScoringResultRow",
+    "ScoreBatchResponse",
+    "ScoreResponse",
+    "ScoreRequest",
+    "ScoreBatchRequest",
+    "AggregationFunctionType",
+    "BasicScoringFnParams",
+    "CommonScoringFnFields",
+    "LLMAsJudgeScoringFnParams",
+    "ListScoringFunctionsResponse",
+    "RegexParserScoringFnParams",
+    "ScoringFn",
+    "ScoringFnInput",
+    "ScoringFnParams",
+    "ScoringFnParamsType",
+    "ScoringFunctions",
+    "fastapi_routes",
+]

llama_stack_api/scoring/api.py

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Scoring API protocol definition.
+
+This module contains the Scoring protocol definition.
+Pydantic models are defined in llama_stack_api.scoring.models.
+The FastAPI router is defined in llama_stack_api.scoring.fastapi_routes.
+"""
+
+from typing import Protocol, runtime_checkable
+
+from llama_stack_api.scoring_functions import ScoringFn
+
+from .models import ScoreBatchRequest, ScoreBatchResponse, ScoreRequest, ScoreResponse
+
+
+class ScoringFunctionStore(Protocol):
+    """Protocol for storing and retrieving scoring functions."""
+
+    def get_scoring_function(self, scoring_fn_id: str) -> ScoringFn: ...
+
+
+@runtime_checkable
+class Scoring(Protocol):
+    """Protocol for scoring operations."""
+
+    scoring_function_store: ScoringFunctionStore
+
+    async def score_batch(self, request: ScoreBatchRequest) -> ScoreBatchResponse: ...
+
+    async def score(self, request: ScoreRequest) -> ScoreResponse: ...

llama_stack_api/scoring/fastapi_routes.py

@@ -0,0 +1,67 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""FastAPI router for the Scoring API.
+
+This module defines the FastAPI router for the Scoring API using standard
+FastAPI route decorators.
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body
+
+from llama_stack_api.router_utils import standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1
+
+from .api import Scoring
+from .models import ScoreBatchRequest, ScoreBatchResponse, ScoreRequest, ScoreResponse
+
+
+def create_router(impl: Scoring) -> APIRouter:
+    """Create a FastAPI router for the Scoring API.
+
+    Args:
+        impl: The Scoring implementation instance
+
+    Returns:
+        APIRouter configured for the Scoring API
+    """
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1}",
+        tags=["Scoring"],
+        responses=standard_responses,
+    )
+
+    @router.post(
+        "/scoring/score",
+        response_model=ScoreResponse,
+        summary="Score a list of rows.",
+        description="Score a list of rows.",
+        responses={
+            200: {"description": "A ScoreResponse object containing rows and aggregated results."},
+        },
+    )
+    async def score(
+        request: Annotated[ScoreRequest, Body(...)],
+    ) -> ScoreResponse:
+        return await impl.score(request)
+
+    @router.post(
+        "/scoring/score-batch",
+        response_model=ScoreBatchResponse,
+        summary="Score a batch of rows.",
+        description="Score a batch of rows.",
+        responses={
+            200: {"description": "A ScoreBatchResponse."},
+        },
+    )
+    async def score_batch(
+        request: Annotated[ScoreBatchRequest, Body(...)],
+    ) -> ScoreBatchResponse:
+        return await impl.score_batch(request)
+
+    return router

llama_stack_api/scoring/models.py

@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Pydantic models for Scoring API requests and responses.
+
+This module defines the request and response models for the Scoring API
+using Pydantic with Field descriptions for OpenAPI schema generation.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from llama_stack_api.schema_utils import json_schema_type
+from llama_stack_api.scoring_functions import ScoringFnParams
+
+# mapping of metric to value
+ScoringResultRow = dict[str, Any]
+
+
+@json_schema_type
+class ScoringResult(BaseModel):
+    """
+    A scoring result for a single row.
+    """
+
+    score_rows: list[ScoringResultRow] = Field(
+        ..., description="The scoring result for each row. Each row is a map of column name to value."
+    )
+    aggregated_results: dict[str, Any] = Field(..., description="Map of metric name to aggregated value")
+
+
+@json_schema_type
+class ScoreBatchResponse(BaseModel):
+    """Response from batch scoring operations on datasets."""
+
+    dataset_id: str | None = Field(default=None, description="(Optional) The identifier of the dataset that was scored")
+    results: dict[str, ScoringResult] = Field(..., description="A map of scoring function name to ScoringResult")
+
+
+@json_schema_type
+class ScoreResponse(BaseModel):
+    """
+    The response from scoring.
+    """
+
+    results: dict[str, ScoringResult] = Field(..., description="A map of scoring function name to ScoringResult.")
+
+
+@json_schema_type
+class ScoreRequest(BaseModel):
+    """Request model for scoring a list of rows."""
+
+    input_rows: list[dict[str, Any]] = Field(..., description="The rows to score.")
+    scoring_functions: dict[str, ScoringFnParams | None] = Field(
+        ..., description="The scoring functions to use for the scoring."
+    )
+
+
+@json_schema_type
+class ScoreBatchRequest(BaseModel):
+    """Request model for scoring a batch of rows from a dataset."""
+
+    dataset_id: str = Field(..., description="The ID of the dataset to score.")
+    scoring_functions: dict[str, ScoringFnParams | None] = Field(
+        ..., description="The scoring functions to use for the scoring."
+    )
+    save_results_dataset: bool = Field(default=False, description="Whether to save the results to a dataset.")
+
+
+__all__ = [
+    "ScoringResult",
+    "ScoringResultRow",
+    "ScoreBatchResponse",
+    "ScoreResponse",
+    "ScoreRequest",
+    "ScoreBatchRequest",
+]

llama_stack_api/scoring_functions/__init__.py

@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""ScoringFunctions API protocol and models.
+
+This module contains the ScoringFunctions protocol definition.
+Pydantic models are defined in llama_stack_api.scoring_functions.models.
+The FastAPI router is defined in llama_stack_api.scoring_functions.fastapi_routes.
+"""
+
+from . import fastapi_routes
+from .api import ScoringFunctions
+from .models import (
+    AggregationFunctionType,
+    BasicScoringFnParams,
+    CommonScoringFnFields,
+    GetScoringFunctionRequest,
+    ListScoringFunctionsRequest,
+    ListScoringFunctionsResponse,
+    LLMAsJudgeScoringFnParams,
+    RegexParserScoringFnParams,
+    RegisterScoringFunctionRequest,
+    ScoringFn,
+    ScoringFnInput,
+    ScoringFnParams,
+    ScoringFnParamsType,
+    UnregisterScoringFunctionRequest,
+)
+
+__all__ = [
+    "ScoringFunctions",
+    "ScoringFn",
+    "ScoringFnInput",
+    "ScoringFnParams",
+    "ScoringFnParamsType",
+    "AggregationFunctionType",
+    "LLMAsJudgeScoringFnParams",
+    "RegexParserScoringFnParams",
+    "BasicScoringFnParams",
+    "CommonScoringFnFields",
+    "ListScoringFunctionsResponse",
+    "ListScoringFunctionsRequest",
+    "GetScoringFunctionRequest",
+    "RegisterScoringFunctionRequest",
+    "UnregisterScoringFunctionRequest",
+    "fastapi_routes",
+]

llama_stack_api/scoring_functions/api.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Protocol, runtime_checkable
+
+from .models import (
+    GetScoringFunctionRequest,
+    ListScoringFunctionsRequest,
+    ListScoringFunctionsResponse,
+    RegisterScoringFunctionRequest,
+    ScoringFn,
+    UnregisterScoringFunctionRequest,
+)
+
+
+@runtime_checkable
+class ScoringFunctions(Protocol):
+    async def list_scoring_functions(
+        self,
+        request: ListScoringFunctionsRequest,
+    ) -> ListScoringFunctionsResponse: ...
+
+    async def get_scoring_function(
+        self,
+        request: GetScoringFunctionRequest,
+    ) -> ScoringFn: ...
+
+    async def register_scoring_function(
+        self,
+        request: RegisterScoringFunctionRequest,
+    ) -> None: ...
+
+    async def unregister_scoring_function(
+        self,
+        request: UnregisterScoringFunctionRequest,
+    ) -> None: ...

llama_stack_api/scoring_functions/fastapi_routes.py

@@ -0,0 +1,108 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""FastAPI router for the ScoringFunctions API.
+
+This module defines the FastAPI router for the ScoringFunctions API using standard
+FastAPI route decorators.
+
+The router is defined in the API package to keep all API-related code together.
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+
+from llama_stack_api.router_utils import create_path_dependency, create_query_dependency, standard_responses
+from llama_stack_api.version import LLAMA_STACK_API_V1
+
+from .api import ScoringFunctions
+from .models import (
+    GetScoringFunctionRequest,
+    ListScoringFunctionsRequest,
+    ListScoringFunctionsResponse,
+    RegisterScoringFunctionRequest,
+    ScoringFn,
+    UnregisterScoringFunctionRequest,
+)
+
+get_list_scoring_functions_request = create_query_dependency(ListScoringFunctionsRequest)
+get_get_scoring_function_request = create_path_dependency(GetScoringFunctionRequest)
+get_unregister_scoring_function_request = create_path_dependency(UnregisterScoringFunctionRequest)
+
+
+def create_router(impl: ScoringFunctions) -> APIRouter:
+    """Create a FastAPI router for the ScoringFunctions API.
+
+    Args:
+        impl: The ScoringFunctions implementation instance
+
+    Returns:
+        APIRouter configured for the ScoringFunctions API
+    """
+    router = APIRouter(
+        prefix=f"/{LLAMA_STACK_API_V1}",
+        tags=["Scoring Functions"],
+        responses=standard_responses,
+    )
+
+    @router.get(
+        "/scoring-functions",
+        response_model=ListScoringFunctionsResponse,
+        summary="List all scoring functions.",
+        description="List all scoring functions.",
+        responses={
+            200: {"description": "A ListScoringFunctionsResponse."},
+        },
+    )
+    async def list_scoring_functions(
+        request: Annotated[ListScoringFunctionsRequest, Depends(get_list_scoring_functions_request)],
+    ) -> ListScoringFunctionsResponse:
+        return await impl.list_scoring_functions(request)
+
+    @router.get(
+        "/scoring-functions/{scoring_fn_id:path}",
+        response_model=ScoringFn,
+        summary="Get a scoring function by its ID.",
+        description="Get a scoring function by its ID.",
+        responses={
+            200: {"description": "A ScoringFn."},
+        },
+    )
+    async def get_scoring_function(
+        request: Annotated[GetScoringFunctionRequest, Depends(get_get_scoring_function_request)],
+    ) -> ScoringFn:
+        return await impl.get_scoring_function(request)
+
+    @router.post(
+        "/scoring-functions",
+        summary="Register a scoring function.",
+        description="Register a scoring function.",
+        responses={
+            200: {"description": "The scoring function was successfully registered."},
+        },
+        deprecated=True,
+    )
+    async def register_scoring_function(
+        request: Annotated[RegisterScoringFunctionRequest, Body(...)],
+    ) -> None:
+        return await impl.register_scoring_function(request)
+
+    @router.delete(
+        "/scoring-functions/{scoring_fn_id:path}",
+        summary="Unregister a scoring function.",
+        description="Unregister a scoring function.",
+        responses={
+            200: {"description": "The scoring function was successfully unregistered."},
+        },
+        deprecated=True,
+    )
+    async def unregister_scoring_function(
+        request: Annotated[UnregisterScoringFunctionRequest, Depends(get_unregister_scoring_function_request)],
+    ) -> None:
+        return await impl.unregister_scoring_function(request)
+
+    return router

llama_stack_api/scoring_functions/models.py

@@ -0,0 +1,214 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Pydantic models for ScoringFunctions API requests and responses.
+
+This module defines the request and response models for the ScoringFunctions API
+using Pydantic with Field descriptions for OpenAPI schema generation.
+"""
+
+from enum import StrEnum
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field
+
+from llama_stack_api.common.type_system import ParamType
+from llama_stack_api.resource import Resource, ResourceType
+from llama_stack_api.schema_utils import json_schema_type, register_schema
+
+
+@json_schema_type
+class ScoringFnParamsType(StrEnum):
+    """Types of scoring function parameter configurations.
+    :cvar llm_as_judge: Use an LLM model to evaluate and score responses
+    :cvar regex_parser: Use regex patterns to extract and score specific parts of responses
+    :cvar basic: Basic scoring with simple aggregation functions
+    """
+
+    llm_as_judge = "llm_as_judge"
+    regex_parser = "regex_parser"
+    basic = "basic"
+
+
+@json_schema_type
+class AggregationFunctionType(StrEnum):
+    """Types of aggregation functions for scoring results.
+    :cvar average: Calculate the arithmetic mean of scores
+    :cvar weighted_average: Calculate a weighted average of scores
+    :cvar median: Calculate the median value of scores
+    :cvar categorical_count: Count occurrences of categorical values
+    :cvar accuracy: Calculate accuracy as the proportion of correct answers
+    """
+
+    average = "average"
+    weighted_average = "weighted_average"
+    median = "median"
+    categorical_count = "categorical_count"
+    accuracy = "accuracy"
+
+
+@json_schema_type
+class LLMAsJudgeScoringFnParams(BaseModel):
+    """Parameters for LLM-as-judge scoring function configuration.
+    :param type: The type of scoring function parameters, always llm_as_judge
+    :param judge_model: Identifier of the LLM model to use as a judge for scoring
+    :param prompt_template: (Optional) Custom prompt template for the judge model
+    :param judge_score_regexes: Regexes to extract the answer from generated response
+    :param aggregation_functions: Aggregation functions to apply to the scores of each row
+    """
+
+    type: Literal[ScoringFnParamsType.llm_as_judge] = ScoringFnParamsType.llm_as_judge
+    judge_model: str
+    prompt_template: str | None = None
+    judge_score_regexes: list[str] = Field(
+        description="Regexes to extract the answer from generated response",
+        default_factory=lambda: [],
+    )
+    aggregation_functions: list[AggregationFunctionType] = Field(
+        description="Aggregation functions to apply to the scores of each row",
+        default_factory=lambda: [],
+    )
+
+
+@json_schema_type
+class RegexParserScoringFnParams(BaseModel):
+    """Parameters for regex parser scoring function configuration.
+    :param type: The type of scoring function parameters, always regex_parser
+    :param parsing_regexes: Regex to extract the answer from generated response
+    :param aggregation_functions: Aggregation functions to apply to the scores of each row
+    """
+
+    type: Literal[ScoringFnParamsType.regex_parser] = ScoringFnParamsType.regex_parser
+    parsing_regexes: list[str] = Field(
+        description="Regex to extract the answer from generated response",
+        default_factory=lambda: [],
+    )
+    aggregation_functions: list[AggregationFunctionType] = Field(
+        description="Aggregation functions to apply to the scores of each row",
+        default_factory=lambda: [],
+    )
+
+
+@json_schema_type
+class BasicScoringFnParams(BaseModel):
+    """Parameters for basic scoring function configuration.
+    :param type: The type of scoring function parameters, always basic
+    :param aggregation_functions: Aggregation functions to apply to the scores of each row
+    """
+
+    type: Literal[ScoringFnParamsType.basic] = ScoringFnParamsType.basic
+    aggregation_functions: list[AggregationFunctionType] = Field(
+        description="Aggregation functions to apply to the scores of each row",
+        default_factory=list,
+    )
+
+
+ScoringFnParams = Annotated[
+    LLMAsJudgeScoringFnParams | RegexParserScoringFnParams | BasicScoringFnParams,
+    Field(discriminator="type"),
+]
+register_schema(ScoringFnParams, name="ScoringFnParams")
+
+
+@json_schema_type
+class ListScoringFunctionsRequest(BaseModel):
+    """Request model for listing scoring functions."""
+
+    pass
+
+
+@json_schema_type
+class GetScoringFunctionRequest(BaseModel):
+    """Request model for getting a scoring function."""
+
+    scoring_fn_id: str = Field(..., description="The ID of the scoring function to get.")
+
+
+@json_schema_type
+class RegisterScoringFunctionRequest(BaseModel):
+    """Request model for registering a scoring function."""
+
+    scoring_fn_id: str = Field(..., description="The ID of the scoring function to register.")
+    description: str = Field(..., description="The description of the scoring function.")
+    return_type: ParamType = Field(..., description="The return type of the scoring function.")
+    provider_scoring_fn_id: str | None = Field(
+        default=None, description="The ID of the provider scoring function to use for the scoring function."
+    )
+    provider_id: str | None = Field(default=None, description="The ID of the provider to use for the scoring function.")
+    params: ScoringFnParams | None = Field(
+        default=None,
+        description="The parameters for the scoring function for benchmark eval, these can be overridden for app eval.",
+    )
+
+
+@json_schema_type
+class UnregisterScoringFunctionRequest(BaseModel):
+    """Request model for unregistering a scoring function."""
+
+    scoring_fn_id: str = Field(..., description="The ID of the scoring function to unregister.")
+
+
+class CommonScoringFnFields(BaseModel):
+    description: str | None = None
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Any additional metadata for this definition",
+    )
+    return_type: ParamType = Field(
+        description="The return type of the deterministic function",
+    )
+    params: ScoringFnParams | None = Field(
+        description="The parameters for the scoring function for benchmark eval, these can be overridden for app eval",
+        default=None,
+    )
+
+
+@json_schema_type
+class ScoringFn(CommonScoringFnFields, Resource):
+    """A scoring function resource for evaluating model outputs.
+    :param type: The resource type, always scoring_function
+    """
+
+    type: Literal[ResourceType.scoring_function] = ResourceType.scoring_function
+
+    @property
+    def scoring_fn_id(self) -> str:
+        return self.identifier
+
+    @property
+    def provider_scoring_fn_id(self) -> str | None:
+        return self.provider_resource_id
+
+
+class ScoringFnInput(CommonScoringFnFields, BaseModel):
+    scoring_fn_id: str
+    provider_id: str | None = None
+    provider_scoring_fn_id: str | None = None
+
+
+@json_schema_type
+class ListScoringFunctionsResponse(BaseModel):
+    """Response containing a list of scoring function objects."""
+
+    data: list[ScoringFn] = Field(..., description="List of scoring function objects.")
+
+
+__all__ = [
+    "ScoringFnParamsType",
+    "AggregationFunctionType",
+    "LLMAsJudgeScoringFnParams",
+    "RegexParserScoringFnParams",
+    "BasicScoringFnParams",
+    "ScoringFnParams",
+    "ListScoringFunctionsRequest",
+    "GetScoringFunctionRequest",
+    "RegisterScoringFunctionRequest",
+    "UnregisterScoringFunctionRequest",
+    "CommonScoringFnFields",
+    "ScoringFn",
+    "ScoringFnInput",
+    "ListScoringFunctionsResponse",
+]