guidellm 0.3.1__py3-none-any.whl → 0.6.0a5__py3-none-any.whl

guidellm/mock_server/server.py

@@ -0,0 +1,238 @@
+"""
+High-performance mock server for OpenAI and vLLM API compatibility testing.
+
+This module provides a Sanic-based mock server that simulates OpenAI and vLLM APIs
+with configurable latency, token generation patterns, and response characteristics.
+The server supports both streaming and non-streaming endpoints, enabling realistic
+performance testing and validation of GuideLLM benchmarking workflows without
+requiring actual model deployments.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, cast
+
+from sanic import Sanic, response
+from sanic.exceptions import NotFound
+from sanic.log import logger
+from sanic.request import File, Request
+from sanic.response import BaseHTTPResponse, HTTPResponse
+
+from guidellm.mock_server.config import MockServerConfig
+from guidellm.mock_server.handlers import (
+    ChatCompletionsHandler,
+    CompletionsHandler,
+    TokenizerHandler,
+)
+
+__all__ = ["MockServer"]
+
+
+class MockServer:
+    """
+    High-performance mock server implementing OpenAI and vLLM API endpoints.
+
+    Provides a Sanic-based web server that simulates API responses with configurable
+    timing characteristics for testing and benchmarking purposes. Supports chat
+    completions, text completions, tokenization endpoints, and model listing with
+    realistic latency patterns to enable comprehensive performance validation.
+
+    Example:
+    ::
+        config = ServerConfig(model="test-model", port=8080)
+        server = MockServer(config)
+        server.run()
+    """
+
+    def __init__(self, config: MockServerConfig) -> None:
+        """
+        Initialize the mock server with configuration.
+
+        :param config: Server configuration containing network settings and response
+            timing parameters
+        """
+        self.config = config
+        self.app = Sanic("guidellm-mock-server")
+        self.chat_handler = ChatCompletionsHandler(config)
+        self.completions_handler = CompletionsHandler(config)
+        self.tokenizer_handler = TokenizerHandler(config)
+
+        self._setup_middleware()
+        self._setup_routes()
+        self._setup_error_handlers()
+
+    def _setup_middleware(self):
+        """Setup middleware for CORS, logging, etc."""
+
+        @self.app.middleware("request")
+        async def add_cors_headers(_request: Request) -> None:
+            """Add CORS headers to all requests."""
+            return None  # noqa: RET501
+
+        @self.app.middleware("response")
+        async def add_response_headers(
+            _request: Any, resp: BaseHTTPResponse
+        ) -> HTTPResponse:
+            """Add standard response headers."""
+            resp.headers["Access-Control-Allow-Origin"] = "*"
+            resp.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+            resp.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
+            resp.headers["Server"] = "guidellm-mock-server"
+            return resp  # type: ignore[return-value]
+
+    def _setup_routes(self):  # noqa: C901
+        @self.app.get("/health")
+        async def health_check(_request: Request):
+            return response.json({"status": "healthy", "timestamp": time.time()})
+
+        @self.app.get("/v1/models")
+        async def list_models(_request: Request):
+            return response.json(
+                {
+                    "object": "list",
+                    "data": [
+                        {
+                            "id": self.config.model,
+                            "object": "model",
+                            "created": int(time.time()),
+                            "owned_by": "guidellm-mock",
+                        }
+                    ],
+                }
+            )
+
+        @self.app.route("/v1/chat/completions", methods=["POST", "OPTIONS"])
+        async def chat_completions(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.chat_handler.handle(request)
+
+        @self.app.route("/v1/completions", methods=["POST", "OPTIONS"])
+        async def completions(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.completions_handler.handle(request)
+
+        @self.app.route("/tokenize", methods=["POST", "OPTIONS"])
+        async def tokenize(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.tokenizer_handler.tokenize(request)
+
+        @self.app.route("/detokenize", methods=["POST", "OPTIONS"])
+        async def detokenize(request: Request):
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            return await self.tokenizer_handler.detokenize(request)
+
+        @self.app.route("/v1/audio/transcriptions", methods=["POST", "OPTIONS"])
+        async def audio_transcriptions(request: Request) -> HTTPResponse:
+            """
+            Mock OpenAI audio transcription endpoint:
+            - receives multipart/form-data
+            - file field contains audio file
+            - model field is optional, default to "mock-model"
+            - returns "transcribed text"
+            """
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            if request.files is None or request.form is None:
+                return response.json({"error": "No form data provided"}, status=400)
+            file: File | None = request.files.get("file")
+            if "file" not in request.files or "model" not in request.form:
+                return response.json(
+                    {"error": "Missing 'file' in form-data"}, status=400
+                )
+
+            file = cast("File", file)
+            model = request.form.get("model", "mock-model")
+
+            return response.json(
+                {
+                    "text": f"Mock transcription for {file.name}",
+                    "file_size": len(file.body),
+                    "model_used": model,
+                    "transcription": f"Transcribed({file.name}) using {model}",
+                }
+            )
+
+        @self.app.route("/v1/audio/translations", methods=["POST", "OPTIONS"])
+        async def audio_translations(request: Request) -> HTTPResponse:
+            """
+            Mock OpenAI audio translation endpoint:
+            - receives multipart/form-data
+            - file field contains audio file
+            - model field is optional, default to "mock-model"
+            - returns translated text
+            """
+            if request.method == "OPTIONS":
+                return response.text("", status=204)
+            if request.files is None or request.form is None:
+                return response.json({"error": "No form data provided"}, status=400)
+            file: File | None = request.files.get("file")
+            if "file" not in request.files or "model" not in request.form:
+                return response.json(
+                    {"error": "Missing 'file' in form-data"}, status=400
+                )
+
+            file = cast("File", file)
+            decoded_text = (
+                "This is a mock translation result."  # mock output tranlated text
+            )
+
+            return response.json(
+                {
+                    "text": decoded_text,
+                    "file_size": len(file.body),
+                    "filename": {file.name},
+                    "model_used": request.form.get("model", "mock-model"),
+                    "mimetype": file.type,
+                }
+            )
+
+    def _setup_error_handlers(self):
+        """Setup error handlers."""
+
+        @self.app.exception(Exception)
+        async def generic_error_handler(_request: Request, exception: Exception):
+            logger.error(f"Unhandled exception: {exception}")
+            return response.json(
+                {
+                    "error": {
+                        "message": "Internal server error",
+                        "type": type(exception).__name__,
+                        "error": str(exception),
+                    }
+                },
+                status=500,
+            )
+
+        @self.app.exception(NotFound)
+        async def not_found_handler(_request: Request, _exception):
+            return response.json(
+                {
+                    "error": {
+                        "message": "Not Found",
+                        "type": "not_found_error",
+                        "code": "not_found",
+                    }
+                },
+                status=404,
+            )
+
+    def run(self) -> None:
+        """
+        Start the mock server with configured settings.
+
+        Runs the Sanic application in single-process mode with access logging enabled
+        for debugging and monitoring request patterns during testing.
+        """
+        self.app.run(
+            host=self.config.host,
+            port=self.config.port,
+            debug=False,
+            single_process=True,
+            access_log=True,
+            register_sys_signals=False,  # Disable signal handlers for threading
+        )

guidellm/mock_server/utils.py

@@ -0,0 +1,302 @@
+"""
+Mock server utilities for text generation and tokenization testing.
+
+This module provides mock tokenization and text generation utilities for testing
+guidellm's mock server functionality. It includes a mock tokenizer that simulates
+tokenization processes, functions to generate reproducible fake text with specific
+token counts, and timing generators for realistic benchmarking scenarios.
+"""
+
+from __future__ import annotations
+
+import random
+import re
+from collections.abc import Generator
+
+from faker import Faker
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer, TextInput
+
+__all__ = [
+    "MockTokenizer",
+    "create_fake_text",
+    "create_fake_tokens_str",
+    "sample_number",
+    "times_generator",
+]
+
+
+class MockTokenizer(PreTrainedTokenizer):
+    """
+    Mock tokenizer implementation for testing text processing workflows.
+
+    Provides a simplified tokenizer that splits text using regex patterns and
+    generates deterministic token IDs based on string hashing. Used for testing
+    guidellm components without requiring actual model tokenizers.
+
+    :cvar VocabSize: Fixed vocabulary size for the mock tokenizer
+    """
+
+    VocabSize = 100000007
+
+    def __len__(self) -> int:
+        """
+        Get the vocabulary size of the tokenizer.
+
+        :return: The total number of tokens in the vocabulary
+        """
+        return self.VocabSize
+
+    def __call__(self, text: str | list[str], **kwargs) -> list[int]:  # noqa: ARG002
+        """
+        Tokenize text and return token IDs (callable interface).
+
+        :param text: Input text to tokenize
+        :return: List of token IDs
+        """
+        if isinstance(text, str):
+            tokens = self.tokenize(text)
+            return self.convert_tokens_to_ids(tokens)
+        elif isinstance(text, list):
+            # Handle batch processing
+            result = []
+            for t in text:
+                result.extend(self.__call__(t))
+            return result
+        else:
+            msg = f"text input must be of type `str` or `list[str]`, got {type(text)}"
+            raise ValueError(msg)
+
+    def tokenize(self, text: TextInput, **_kwargs) -> list[str]:  # type: ignore[override]
+        """
+        Tokenize input text into a list of token strings.
+
+        Splits text using regex to separate words, punctuation, and whitespace
+        into individual tokens for processing.
+
+        :param text: Input text to tokenize
+        :return: List of token strings from the input text
+        """
+        # Split text into tokens: words, spaces, and punctuation
+        return re.findall(r"\w+|[^\w\s]|\s+", text)
+
+    def convert_tokens_to_ids(self, tokens: str | list[str]) -> list[int]:
+        """
+        Convert token strings to numeric token IDs.
+
+        Uses deterministic hashing to generate consistent token IDs for
+        reproducible testing scenarios.
+
+        :param tokens: Single token string or list of token strings
+        :return: Single token ID or list of token IDs
+        """
+        if isinstance(tokens, str):
+            return [hash(tokens) % self.VocabSize]
+        return [hash(token) % self.VocabSize for token in tokens]
+
+    def convert_ids_to_tokens(  # type: ignore[override]
+        self, ids: list[int], _skip_special_tokens: bool = False
+    ) -> list[str]:
+        """
+        Convert numeric token IDs back to token strings.
+
+        Generates fake text tokens using Faker library seeded with token IDs
+        for deterministic and reproducible token generation.
+
+        :param ids: Single token ID or list of token IDs to convert
+        :return: Single token string or list of token strings
+        """
+        if not ids:
+            return [""]
+
+        fake = Faker()
+        fake.seed_instance(sum(ids) % self.VocabSize)
+
+        target_count = len(ids)
+        current_count = 0
+        tokens = []
+
+        while current_count < target_count:
+            text = fake.text(
+                max_nb_chars=(target_count - current_count) * 10  # oversample
+            )
+            new_tokens = self.tokenize(text)
+
+            if current_count > 0:
+                new_tokens = [".", " "] + new_tokens
+
+            new_tokens = (
+                new_tokens[: target_count - current_count]
+                if len(new_tokens) > (target_count - current_count)
+                else new_tokens
+            )
+            tokens += new_tokens
+            current_count += len(new_tokens)
+
+        return tokens
+
+    def convert_tokens_to_string(self, tokens: list[str]) -> str:
+        """
+        Convert a list of token strings back to a single text string.
+
+        :param tokens: List of token strings to concatenate
+        :return: Concatenated string from all tokens
+        """
+        return "".join(tokens)
+
+    def _add_tokens(
+        self,
+        new_tokens: list[str] | list[AddedToken],  # noqa: ARG002
+        special_tokens: bool = False,  # noqa: ARG002
+    ) -> int:
+        """
+        Add new tokens to the tokenizer vocabulary (mock implementation).
+
+        :param new_tokens: List of tokens to add to the vocabulary
+        :param special_tokens: Whether the tokens are special tokens
+        :return: Number of tokens actually added (always 0 for mock)
+        """
+        return 0
+
+    def apply_chat_template(  # type: ignore[override]
+        self,
+        conversation: list,
+        tokenize: bool = False,  # Changed default to False to match transformers
+        add_generation_prompt: bool = False,  # noqa: ARG002
+        **kwargs,  # noqa: ARG002
+    ) -> str | list[int]:
+        """
+        Apply a chat template to format conversation messages.
+
+        Mock implementation that concatenates all message content for testing.
+
+        :param conversation: List of chat messages
+        :param tokenize: Whether to return tokens or string
+        :param add_generation_prompt: Whether to add generation prompt
+        :return: Formatted text string or token IDs
+        """
+        # Simple concatenation of all message content
+        texts = []
+        for message in conversation:
+            if isinstance(message, dict) and "content" in message:
+                texts.append(message["content"])
+            elif hasattr(message, "content"):
+                texts.append(message.content)
+
+        formatted_text = " ".join(texts)
+
+        if tokenize:
+            return self.convert_tokens_to_ids(self.tokenize(formatted_text))
+        return formatted_text
+
+    def decode(  # type: ignore[override]
+        self,
+        token_ids: list[int],
+        skip_special_tokens: bool = True,
+        **kwargs,  # noqa: ARG002
+    ) -> str:
+        """
+        Decode token IDs back to text string.
+
+        :param token_ids: List of token IDs to decode
+        :param skip_special_tokens: Whether to skip special tokens
+        :return: Decoded text string
+        """
+        tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens)
+        return self.convert_tokens_to_string(tokens)
+
+
+def create_fake_text(
+    num_tokens: int,
+    processor: PreTrainedTokenizer,
+    seed: int = 42,
+    fake: Faker | None = None,
+) -> str:
+    """
+    Generate fake text using a tokenizer processor with specified token count.
+
+    Creates text by generating fake tokens and joining them into a string,
+    ensuring the result has the exact number of tokens when processed by
+    the given tokenizer.
+
+    :param num_tokens: Target number of tokens in the generated text
+    :param processor: Tokenizer to use for token generation and validation
+    :param seed: Random seed for reproducible text generation
+    :param fake: Optional Faker instance for text generation
+    :return: Generated text string with the specified token count
+    """
+    return "".join(create_fake_tokens_str(num_tokens, processor, seed, fake))
+
+
+def create_fake_tokens_str(
+    num_tokens: int,
+    processor: PreTrainedTokenizer,
+    seed: int = 42,
+    fake: Faker | None = None,
+) -> list[str]:
+    """
+    Generate fake token strings using a tokenizer processor.
+
+    Creates a list of token strings by generating fake text and tokenizing it
+    until the desired token count is reached. Uses the provided tokenizer
+    for accurate token boundary detection.
+
+    :param num_tokens: Target number of tokens to generate
+    :param processor: Tokenizer to use for token generation and validation
+    :param seed: Random seed for reproducible token generation
+    :param fake: Optional Faker instance for text generation
+    :return: List of token strings with the specified count
+    """
+    if not fake:
+        fake = Faker()
+    fake.seed_instance(seed)
+
+    tokens: list[str] = []
+
+    while len(tokens) < num_tokens:
+        text = fake.text(
+            max_nb_chars=(num_tokens - len(tokens)) * 30  # oversample
+        )
+        new_tokens = processor.tokenize(text)
+
+        if len(tokens) > 0:
+            new_tokens = [".", " "] + new_tokens
+
+        new_tokens = (
+            new_tokens[: num_tokens - len(tokens)]
+            if len(new_tokens) > (num_tokens - len(tokens))
+            else new_tokens
+        )
+        tokens += new_tokens
+
+    return tokens
+
+
+def times_generator(mean: float, standard_dev: float) -> Generator[float]:
+    """
+    Generate infinite timing values from a normal distribution.
+
+    Creates a generator that yields timing values sampled from a normal
+    distribution, useful for simulating realistic request timing patterns
+    in benchmarking scenarios.
+
+    :param mean: Mean value for the normal distribution
+    :param standard_dev: Standard deviation for the normal distribution
+    :return: Generator yielding positive timing values from the distribution
+    """
+    while True:
+        yield sample_number(mean, standard_dev)
+
+
+def sample_number(mean: float, standard_dev: float) -> float:
+    """
+    Generate a single timing value from a normal distribution.
+
+    Samples one timing value from a normal distribution with the specified
+    parameters, ensuring the result is non-negative for realistic timing
+    simulation in benchmarking scenarios.
+
+    :param mean: Mean value for the normal distribution
+    :param standard_dev: Standard deviation for the normal distribution
+    :return: Non-negative timing value from the distribution
+    """
+    return max(0.0, random.gauss(mean, standard_dev))

guidellm/scheduler/__init__.py

@@ -1,47 +1,90 @@
-from .result import (
-    SchedulerRequestInfo,
-    SchedulerRequestResult,
-    SchedulerResult,
-    SchedulerRunInfo,
+"""
+Scheduler subsystem for orchestrating benchmark workloads and managing worker processes.
+
+This module provides the core scheduling infrastructure for guidellm, including
+strategies for controlling request timing patterns (synchronous, asynchronous,
+constant rate, Poisson), constraints for limiting benchmark execution (duration,
+error rates, request counts), and distributed execution through worker processes.
+The scheduler coordinates between backend interfaces, manages benchmark state
+transitions, and handles multi-turn request sequences with customizable timing
+strategies and resource constraints.
+"""
+
+from .constraints import (
+    Constraint,
+    ConstraintInitializer,
+    ConstraintsInitializerFactory,
+    MaxDurationConstraint,
+    MaxErrorRateConstraint,
+    MaxErrorsConstraint,
+    MaxGlobalErrorRateConstraint,
+    MaxNumberConstraint,
+    OverSaturationConstraint,
+    OverSaturationConstraintInitializer,
+    PydanticConstraintInitializer,
+    SerializableConstraintInitializer,
+    UnserializableConstraintInitializer,
 )
+from .environments import Environment, NonDistributedEnvironment
 from .scheduler import Scheduler
-from .strategy import (
+from .schemas import (
+    BackendInterface,
+    BackendT,
+    MultiTurnRequestT,
+    RequestT,
+    ResponseT,
+    SchedulerMessagingPydanticRegistry,
+    SchedulerProgress,
+    SchedulerState,
+    SchedulerUpdateAction,
+)
+from .strategies import (
     AsyncConstantStrategy,
     AsyncPoissonStrategy,
     ConcurrentStrategy,
     SchedulingStrategy,
+    StrategyT,
     StrategyType,
     SynchronousStrategy,
     ThroughputStrategy,
-    strategy_display_str,
-)
-from .worker import (
-    GenerativeRequestsWorker,
-    GenerativeRequestsWorkerDescription,
-    RequestsWorker,
-    ResolveStatus,
-    WorkerDescription,
-    WorkerProcessResult,
 )
+from .worker import WorkerProcess
+from .worker_group import WorkerProcessGroup
 
 __all__ = [
     "AsyncConstantStrategy",
     "AsyncPoissonStrategy",
+    "BackendInterface",
+    "BackendT",
     "ConcurrentStrategy",
-    "GenerativeRequestsWorker",
-    "GenerativeRequestsWorkerDescription",
-    "RequestsWorker",
-    "ResolveStatus",
+    "Constraint",
+    "ConstraintInitializer",
+    "ConstraintsInitializerFactory",
+    "Environment",
+    "MaxDurationConstraint",
+    "MaxErrorRateConstraint",
+    "MaxErrorsConstraint",
+    "MaxGlobalErrorRateConstraint",
+    "MaxNumberConstraint",
+    "MultiTurnRequestT",
+    "NonDistributedEnvironment",
+    "OverSaturationConstraint",
+    "OverSaturationConstraintInitializer",
+    "PydanticConstraintInitializer",
+    "RequestT",
+    "ResponseT",
     "Scheduler",
-    "SchedulerRequestInfo",
-    "SchedulerRequestResult",
-    "SchedulerResult",
-    "SchedulerRunInfo",
+    "SchedulerMessagingPydanticRegistry",
+    "SchedulerProgress",
+    "SchedulerState",
+    "SchedulerUpdateAction",
     "SchedulingStrategy",
+    "SerializableConstraintInitializer",
+    "StrategyT",
     "StrategyType",
     "SynchronousStrategy",
     "ThroughputStrategy",
-    "WorkerDescription",
-    "WorkerProcessResult",
-    "strategy_display_str",
+    "UnserializableConstraintInitializer",
+    "WorkerProcess",
+    "WorkerProcessGroup",
 ]