guidellm 0.4.0a21__py3-none-any.whl → 0.4.0a169__py3-none-any.whl

guidellm/mock_server/models.py

@@ -0,0 +1,510 @@
+"""
+Pydantic models for OpenAI API and vLLM API request/response validation.
+
+This module defines comprehensive data models for validating and serializing API
+requests and responses compatible with both OpenAI's API specification and vLLM's
+extended parameters. It includes models for chat completions, legacy text completions,
+tokenization operations, and error handling, supporting both streaming and non-streaming
+responses with full type safety and validation.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+__all__ = [
+    "ChatCompletionChoice",
+    "ChatCompletionChunk",
+    "ChatCompletionsRequest",
+    "ChatCompletionsResponse",
+    "ChatMessage",
+    "CompletionChoice",
+    "CompletionsRequest",
+    "CompletionsResponse",
+    "DetokenizeRequest",
+    "DetokenizeResponse",
+    "ErrorDetail",
+    "ErrorResponse",
+    "StreamOptions",
+    "TokenizeRequest",
+    "TokenizeResponse",
+    "Usage",
+]
+
+
+class Usage(BaseModel):
+    """Token usage statistics for API requests and responses.
+
+    Tracks the number of tokens consumed in prompts, completions, and total
+    usage for billing and monitoring purposes.
+    """
+
+    prompt_tokens: int = Field(description="Number of tokens in the input prompt")
+    completion_tokens: int = Field(
+        description="Number of tokens in the generated completion"
+    )
+    total_tokens: int = Field(description="Total tokens used (prompt + completion)")
+
+    def __init__(self, prompt_tokens: int = 0, completion_tokens: int = 0, **kwargs):
+        """Initialize usage statistics.
+
+        :param prompt_tokens: Number of tokens in the input prompt
+        :param completion_tokens: Number of tokens in the generated completion
+        :param kwargs: Additional keyword arguments passed to BaseModel
+        """
+        super().__init__(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+            **kwargs,
+        )
+
+
+class StreamOptions(BaseModel):
+    """Configuration options for streaming API responses.
+
+    Controls the behavior and content of streamed responses including
+    whether to include usage statistics in the final chunk.
+    """
+
+    include_usage: bool | None = Field(
+        default=None,
+        description="Whether to include usage statistics in streaming responses",
+    )
+
+
+class ChatMessage(BaseModel):
+    """A single message in a chat conversation.
+
+    Represents one exchange in a conversational interface with role-based
+    content and optional metadata for advanced features.
+    """
+
+    role: Literal["system", "user", "assistant", "tool"] = Field(
+        description="Role of the message sender in the conversation"
+    )
+    content: str = Field(description="Text content of the message")
+    name: str | None = Field(
+        default=None, description="Optional name identifier for the message sender"
+    )
+
+
+class ChatCompletionsRequest(BaseModel):
+    """Request parameters for chat completion API endpoints.
+
+    Comprehensive model supporting both OpenAI standard parameters and vLLM
+    extensions for advanced generation control, guided decoding, and performance
+    optimization.
+    """
+
+    model: str = Field(description="Model identifier to use for generation")
+    messages: list[ChatMessage] = Field(
+        description="List of messages in the conversation"
+    )
+    max_tokens: int | None = Field(
+        default=None, description="Maximum number of tokens to generate"
+    )
+    max_completion_tokens: int | None = Field(
+        default=None, description="Maximum tokens in completion (OpenAI naming)"
+    )
+    temperature: float | None = Field(
+        default=1.0, description="Sampling temperature for randomness control"
+    )
+    top_p: float | None = Field(default=1.0, description="Nucleus sampling parameter")
+    n: int | None = Field(
+        default=1, description="Number of completion choices to generate"
+    )
+    stream: bool | None = Field(
+        default=False, description="Whether to stream response chunks"
+    )
+    stream_options: StreamOptions | None = Field(
+        default=None, description="Configuration for streaming responses"
+    )
+    stop: str | list[str] | None = Field(
+        default=None, description="Stop sequences to end generation"
+    )
+    presence_penalty: float | None = Field(
+        default=0.0, description="Penalty for token presence to encourage diversity"
+    )
+    frequency_penalty: float | None = Field(
+        default=0.0, description="Penalty for token frequency to reduce repetition"
+    )
+    logit_bias: dict[str, float] | None = Field(
+        default=None, description="Bias values for specific tokens"
+    )
+    seed: int | None = Field(
+        default=None, description="Random seed for reproducible outputs"
+    )
+    user: str | None = Field(
+        default=None, description="User identifier for tracking and abuse monitoring"
+    )
+
+    # vLLM extensions
+    use_beam_search: bool | None = Field(
+        default=False, description="Enable beam search for better quality"
+    )
+    top_k: int | None = Field(default=None, description="Top-k sampling parameter")
+    min_p: float | None = Field(
+        default=None, description="Minimum probability threshold for sampling"
+    )
+    repetition_penalty: float | None = Field(
+        default=None, description="Penalty for repeated tokens"
+    )
+    length_penalty: float | None = Field(
+        default=1.0, description="Length penalty for sequence scoring"
+    )
+    stop_token_ids: list[int] | None = Field(
+        default=None, description="Token IDs that trigger generation stop"
+    )
+    include_stop_str_in_output: bool | None = Field(
+        default=False, description="Include stop sequence in output"
+    )
+    ignore_eos: bool | None = Field(
+        default=False, description="Ignore end-of-sequence tokens"
+    )
+    min_tokens: int | None = Field(
+        default=0, description="Minimum number of tokens to generate"
+    )
+    skip_special_tokens: bool | None = Field(
+        default=True, description="Skip special tokens in output"
+    )
+    spaces_between_special_tokens: bool | None = Field(
+        default=True, description="Add spaces between special tokens"
+    )
+    truncate_prompt_tokens: int | None = Field(
+        default=None, description="Maximum prompt tokens before truncation"
+    )
+    allowed_token_ids: list[int] | None = Field(
+        default=None, description="Restrict generation to specific token IDs"
+    )
+    prompt_logprobs: int | None = Field(
+        default=None, description="Number of logprobs to return for prompt tokens"
+    )
+    add_special_tokens: bool | None = Field(
+        default=True, description="Add special tokens during processing"
+    )
+    guided_json: str | dict[str, Any] | None = Field(
+        default=None, description="JSON schema for guided generation"
+    )
+    guided_regex: str | None = Field(
+        default=None, description="Regex pattern for guided generation"
+    )
+    guided_choice: list[str] | None = Field(
+        default=None, description="List of choices for guided generation"
+    )
+    guided_grammar: str | None = Field(
+        default=None, description="Grammar specification for guided generation"
+    )
+    guided_decoding_backend: str | None = Field(
+        default=None, description="Backend to use for guided decoding"
+    )
+    guided_whitespace_pattern: str | None = Field(
+        default=None, description="Whitespace pattern for guided generation"
+    )
+    priority: int | None = Field(
+        default=0, description="Request priority for scheduling"
+    )
+
+
+class ChatCompletionChoice(BaseModel):
+    """A single completion choice from a chat completion response.
+
+    Contains the generated message and metadata about why generation
+    stopped and the choice's position in the response.
+    """
+
+    index: int = Field(description="Index of this choice in the response")
+    message: ChatMessage = Field(description="Generated message content")
+    finish_reason: Literal["stop", "length", "content_filter", "tool_calls"] | None = (
+        Field(description="Reason why generation finished")
+    )
+
+
+class ChatCompletionsResponse(BaseModel):
+    """Response from chat completion API endpoints.
+
+    Contains generated choices, usage statistics, and metadata for
+    non-streaming chat completion requests.
+    """
+
+    id: str = Field(description="Unique identifier for this completion")
+    object: Literal["chat.completion"] = Field(
+        default="chat.completion", description="Object type identifier"
+    )
+    created: int = Field(
+        default_factory=lambda: int(time.time()),
+        description="Unix timestamp of creation",
+    )
+    model: str = Field(description="Model used for generation")
+    choices: list[ChatCompletionChoice] = Field(
+        description="Generated completion choices"
+    )
+    usage: Usage | None = Field(default=None, description="Token usage statistics")
+    system_fingerprint: str | None = Field(
+        default=None, description="System configuration fingerprint"
+    )
+
+
+class ChatCompletionChunk(BaseModel):
+    """A single chunk in a streamed chat completion response.
+
+    Represents one piece of a streaming response with delta content
+    and optional usage statistics in the final chunk.
+    """
+
+    id: str = Field(description="Unique identifier for this completion")
+    object: Literal["chat.completion.chunk"] = Field(
+        default="chat.completion.chunk",
+        description="Object type identifier for streaming chunks",
+    )
+    created: int = Field(
+        default_factory=lambda: int(time.time()),
+        description="Unix timestamp of creation",
+    )
+    model: str = Field(description="Model used for generation")
+    choices: list[dict[str, Any]] = Field(description="Delta choices for streaming")
+    usage: Usage | None = Field(
+        default=None, description="Token usage statistics (typically in final chunk)"
+    )
+
+
+class CompletionsRequest(BaseModel):
+    """Request parameters for legacy text completion API endpoints.
+
+    Supports the older text completion format with prompt-based input
+    and the same extensive parameter set as chat completions for
+    backward compatibility.
+    """
+
+    model: str = Field(description="Model identifier to use for generation")
+    prompt: str | list[str] = Field(description="Input prompt(s) for completion")
+    max_tokens: int | None = Field(
+        default=16, description="Maximum number of tokens to generate"
+    )
+    temperature: float | None = Field(
+        default=1.0, description="Sampling temperature for randomness control"
+    )
+    top_p: float | None = Field(default=1.0, description="Nucleus sampling parameter")
+    n: int | None = Field(
+        default=1, description="Number of completion choices to generate"
+    )
+    stream: bool | None = Field(
+        default=False, description="Whether to stream response chunks"
+    )
+    stream_options: StreamOptions | None = Field(
+        default=None, description="Configuration for streaming responses"
+    )
+    logprobs: int | None = Field(
+        default=None, description="Number of logprobs to return"
+    )
+    echo: bool | None = Field(
+        default=False, description="Whether to echo the prompt in output"
+    )
+    stop: str | list[str] | None = Field(
+        default_factory=lambda: ["<|endoftext|>"],
+        description="Stop sequences to end generation",
+    )
+    presence_penalty: float | None = Field(
+        default=0.0, description="Penalty for token presence to encourage diversity"
+    )
+    frequency_penalty: float | None = Field(
+        default=0.0, description="Penalty for token frequency to reduce repetition"
+    )
+    best_of: int | None = Field(
+        default=1, description="Number of candidates to generate and return the best"
+    )
+    logit_bias: dict[str, float] | None = Field(
+        default=None, description="Bias values for specific tokens"
+    )
+    seed: int | None = Field(
+        default=None, description="Random seed for reproducible outputs"
+    )
+    suffix: str | None = Field(
+        default=None, description="Suffix to append after completion"
+    )
+    user: str | None = Field(
+        default=None, description="User identifier for tracking and abuse monitoring"
+    )
+
+    # vLLM extensions (same as chat completions)
+    use_beam_search: bool | None = Field(
+        default=False, description="Enable beam search for better quality"
+    )
+    top_k: int | None = Field(default=None, description="Top-k sampling parameter")
+    min_p: float | None = Field(
+        default=None, description="Minimum probability threshold for sampling"
+    )
+    repetition_penalty: float | None = Field(
+        default=None, description="Penalty for repeated tokens"
+    )
+    length_penalty: float | None = Field(
+        default=1.0, description="Length penalty for sequence scoring"
+    )
+    stop_token_ids: list[int] | None = Field(
+        default=None, description="Token IDs that trigger generation stop"
+    )
+    include_stop_str_in_output: bool | None = Field(
+        default=False, description="Include stop sequence in output"
+    )
+    ignore_eos: bool | None = Field(
+        default=False, description="Ignore end-of-sequence tokens"
+    )
+    min_tokens: int | None = Field(
+        default=0, description="Minimum number of tokens to generate"
+    )
+    skip_special_tokens: bool | None = Field(
+        default=True, description="Skip special tokens in output"
+    )
+    spaces_between_special_tokens: bool | None = Field(
+        default=True, description="Add spaces between special tokens"
+    )
+    truncate_prompt_tokens: int | None = Field(
+        default=None, description="Maximum prompt tokens before truncation"
+    )
+    allowed_token_ids: list[int] | None = Field(
+        default=None, description="Restrict generation to specific token IDs"
+    )
+    prompt_logprobs: int | None = Field(
+        default=None, description="Number of logprobs to return for prompt tokens"
+    )
+    add_special_tokens: bool | None = Field(
+        default=True, description="Add special tokens during processing"
+    )
+    guided_json: str | dict[str, Any] | None = Field(
+        default=None, description="JSON schema for guided generation"
+    )
+    guided_regex: str | None = Field(
+        default=None, description="Regex pattern for guided generation"
+    )
+    guided_choice: list[str] | None = Field(
+        default=None, description="List of choices for guided generation"
+    )
+    guided_grammar: str | None = Field(
+        default=None, description="Grammar specification for guided generation"
+    )
+    guided_decoding_backend: str | None = Field(
+        default=None, description="Backend to use for guided decoding"
+    )
+    guided_whitespace_pattern: str | None = Field(
+        default=None, description="Whitespace pattern for guided generation"
+    )
+    priority: int | None = Field(
+        default=0, description="Request priority for scheduling"
+    )
+
+
+class CompletionChoice(BaseModel):
+    """A single completion choice from a text completion response.
+
+    Contains the generated text and metadata about completion
+    quality and stopping conditions.
+    """
+
+    text: str = Field(description="Generated text content")
+    index: int = Field(description="Index of this choice in the response")
+    logprobs: dict[str, Any] | None = Field(
+        default=None, description="Log probabilities for generated tokens"
+    )
+    finish_reason: Literal["stop", "length", "content_filter"] | None = Field(
+        description="Reason why generation finished"
+    )
+
+
+class CompletionsResponse(BaseModel):
+    """Response from legacy text completion API endpoints.
+
+    Contains generated text choices, usage statistics, and metadata
+    for non-streaming text completion requests.
+    """
+
+    id: str = Field(description="Unique identifier for this completion")
+    object: Literal["text_completion"] = Field(
+        default="text_completion", description="Object type identifier"
+    )
+    created: int = Field(
+        default_factory=lambda: int(time.time()),
+        description="Unix timestamp of creation",
+    )
+    model: str = Field(description="Model used for generation")
+    choices: list[CompletionChoice] = Field(description="Generated completion choices")
+    usage: Usage | None = Field(default=None, description="Token usage statistics")
+    system_fingerprint: str | None = Field(
+        default=None, description="System configuration fingerprint"
+    )
+
+
+class TokenizeRequest(BaseModel):
+    """Request for tokenizing text into token sequences.
+
+    Converts input text into model-specific token representations
+    with optional special token handling.
+    """
+
+    text: str = Field(description="Text to tokenize")
+    add_special_tokens: bool | None = Field(
+        default=True, description="Whether to add model-specific special tokens"
+    )
+
+
+class TokenizeResponse(BaseModel):
+    """Response containing tokenized representation of input text.
+
+    Provides both the token sequence and count for analysis
+    and token budget planning.
+    """
+
+    tokens: list[int] = Field(description="List of token IDs")
+    count: int = Field(description="Total number of tokens")
+
+
+class DetokenizeRequest(BaseModel):
+    """Request for converting token sequences back to text.
+
+    Reconstructs human-readable text from model token representations
+    with configurable special token handling.
+    """
+
+    tokens: list[int] = Field(description="List of token IDs to convert")
+    skip_special_tokens: bool | None = Field(
+        default=True, description="Whether to skip special tokens in output"
+    )
+    spaces_between_special_tokens: bool | None = Field(
+        default=True, description="Whether to add spaces between special tokens"
+    )
+
+
+class DetokenizeResponse(BaseModel):
+    """Response containing text reconstructed from tokens.
+
+    Provides the human-readable text representation of the
+    input token sequence.
+    """
+
+    text: str = Field(description="Reconstructed text from tokens")
+
+
+class ErrorDetail(BaseModel):
+    """Detailed error information for API failures.
+
+    Provides structured error data including message, type classification,
+    and optional error codes for debugging and error handling.
+    """
+
+    message: str = Field(description="Human-readable error description")
+    type: str = Field(description="Error type classification")
+    code: str | None = Field(
+        default=None, description="Optional error code for programmatic handling"
+    )
+
+
+class ErrorResponse(BaseModel):
+    """Standardized error response structure for API failures.
+
+    Wraps error details in a consistent format compatible with
+    OpenAI API error response conventions.
+    """
+
+    error: ErrorDetail = Field(description="Detailed error information")

guidellm/mock_server/server.py

@@ -0,0 +1,168 @@
+"""
+High-performance mock server for OpenAI and vLLM API compatibility testing.
+
+This module provides a Sanic-based mock server that simulates OpenAI and vLLM APIs
+with configurable latency, token generation patterns, and response characteristics.
+The server supports both streaming and non-streaming endpoints, enabling realistic
+performance testing and validation of GuideLLM benchmarking workflows without
+requiring actual model deployments.
+"""
+
+from __future__ import annotations
+
+import time
+
+from sanic import Sanic, response
+from sanic.exceptions import NotFound
+from sanic.log import logger
+from sanic.request import Request
+from sanic.response import HTTPResponse
+
+from guidellm.mock_server.config import MockServerConfig
+from guidellm.mock_server.handlers import (
+    ChatCompletionsHandler,
+    CompletionsHandler,
+    TokenizerHandler,
+)
+
+__all__ = ["MockServer"]
+
+
+class MockServer:
+    """
+    High-performance mock server implementing OpenAI and vLLM API endpoints.
+
+    Provides a Sanic-based web server that simulates API responses with configurable
+    timing characteristics for testing and benchmarking purposes. Supports chat
+    completions, text completions, tokenization endpoints, and model listing with
+    realistic latency patterns to enable comprehensive performance validation.
+
+    Example:
+    ::
+        config = ServerConfig(model="test-model", port=8080)
+        server = MockServer(config)
+        server.run()
+    """
+
+    def __init__(self, config: MockServerConfig) -> None:
+        """
+        Initialize the mock server with configuration.
+
+        :param config: Server configuration containing network settings and response
+            timing parameters
+        """
+        self.config = config
+        self.app = Sanic("guidellm-mock-server")
+        self.chat_handler = ChatCompletionsHandler(config)
+        self.completions_handler = CompletionsHandler(config)
+        self.tokenizer_handler = TokenizerHandler(config)
+
+        self._setup_middleware()
+        self._setup_routes()
+        self._setup_error_handlers()
+
+    def _setup_middleware(self):
+        """Setup middleware for CORS, logging, etc."""
+
+        @self.app.middleware("request")
+        async def add_cors_headers(_request: Request):
+            """Add CORS headers to all requests."""
+
+        @self.app.middleware("response")
+        async def add_response_headers(_request: Request, resp: HTTPResponse):
+            """Add standard response headers."""
+            resp.headers["Access-Control-Allow-Origin"] = "*"
+            resp.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+            resp.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
+            resp.headers["Server"] = "guidellm-mock-server"
+
+    def _setup_routes(self):  # noqa: C901
+        @self.app.get("/health")
+        async def health_check(_request: Request):
+            return response.json({"status": "healthy", "timestamp": time.time()})
+
+        @self.app.get("/v1/models")
+        async def list_models(_request: Request):
+            return response.json(
+                {
+                    "object": "list",
+                    "data": [
+                        {
+                            "id": self.config.model,
+                            "object": "model",
+                            "created": int(time.time()),
+                            "owned_by": "guidellm-mock",
+                        }
+                    ],
+                }
+            )
+
+        @self.app.route("/v1/chat/completions", methods=["POST", "OPTIONS"])
+        async def chat_completions(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.chat_handler.handle(request)
+
+        @self.app.route("/v1/completions", methods=["POST", "OPTIONS"])
+        async def completions(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.completions_handler.handle(request)
+
+        @self.app.route("/tokenize", methods=["POST", "OPTIONS"])
+        async def tokenize(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.tokenizer_handler.tokenize(request)
+
+        @self.app.route("/detokenize", methods=["POST", "OPTIONS"])
+        async def detokenize(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.tokenizer_handler.detokenize(request)
+
+    def _setup_error_handlers(self):
+        """Setup error handlers."""
+
+        @self.app.exception(Exception)
+        async def generic_error_handler(_request: Request, exception: Exception):
+            logger.error(f"Unhandled exception: {exception}")
+            return response.json(
+                {
+                    "error": {
+                        "message": "Internal server error",
+                        "type": type(exception).__name__,
+                        "error": str(exception),
+                    }
+                },
+                status=500,
+            )
+
+        @self.app.exception(NotFound)
+        async def not_found_handler(_request: Request, _exception):
+            return response.json(
+                {
+                    "error": {
+                        "message": "Not Found",
+                        "type": "not_found_error",
+                        "code": "not_found",
+                    }
+                },
+                status=404,
+            )
+
+    def run(self) -> None:
+        """
+        Start the mock server with configured settings.
+
+        Runs the Sanic application in single-process mode with access logging enabled
+        for debugging and monitoring request patterns during testing.
+        """
+        self.app.run(
+            host=self.config.host,
+            port=self.config.port,
+            debug=False,
+            single_process=True,
+            access_log=True,
+            register_sys_signals=False,  # Disable signal handlers for threading
+        )