guidellm 0.3.1__py3-none-any.whl → 0.6.0a5__py3-none-any.whl

guidellm/backends/__init__.py

@@ -0,0 +1,33 @@
+"""
+Backend infrastructure for GuideLLM language model interactions.
+
+Provides abstract base classes, concrete backend implementations, and response
+handlers for standardized communication with generative AI model providers.
+The backend system supports distributed execution across worker processes with
+pluggable response handlers for different API formats. Key components include
+the abstract Backend base class, OpenAI-compatible HTTP backend, and response
+handlers for processing streaming and non-streaming API responses.
+"""
+
+from __future__ import annotations
+
+from .backend import Backend, BackendType
+from .openai import OpenAIHTTPBackend
+from .response_handlers import (
+    AudioResponseHandler,
+    ChatCompletionsResponseHandler,
+    GenerationResponseHandler,
+    GenerationResponseHandlerFactory,
+    TextCompletionsResponseHandler,
+)
+
+__all__ = [
+    "AudioResponseHandler",
+    "Backend",
+    "BackendType",
+    "ChatCompletionsResponseHandler",
+    "GenerationResponseHandler",
+    "GenerationResponseHandlerFactory",
+    "OpenAIHTTPBackend",
+    "TextCompletionsResponseHandler",
+]

guidellm/backends/backend.py

@@ -0,0 +1,109 @@
+"""
+Backend interface and registry for generative AI model interactions.
+
+Provides the abstract base class for implementing backends that communicate with
+generative AI models. Backends handle the lifecycle of generation requests and
+provide a standard interface for distributed execution across worker processes.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Literal
+
+from guidellm.scheduler import BackendInterface
+from guidellm.schemas import GenerationRequest, GenerationResponse
+from guidellm.utils import RegistryMixin
+
+__all__ = [
+    "Backend",
+    "BackendType",
+]
+
+
+BackendType = Literal["openai_http"]
+
+
+class Backend(
+    RegistryMixin["type[Backend]"],
+    BackendInterface[GenerationRequest, GenerationResponse],
+):
+    """
+    Base class for generative AI backends with registry and lifecycle management.
+
+    Provides a standard interface for backends that communicate with generative AI
+    models. Combines the registry pattern for automatic discovery with a defined
+    lifecycle for process-based distributed execution. Backend state must be
+    pickleable for distributed execution across process boundaries.
+
+    Backend lifecycle phases:
+    1. Creation and configuration
+    2. Process startup - Initialize resources in worker process
+    3. Validation - Verify backend readiness
+    4. Request resolution - Process generation requests
+    5. Process shutdown - Clean up resources
+
+    Example:
+    ::
+        @Backend.register("my_backend")
+        class MyBackend(Backend):
+            def __init__(self, api_key: str):
+                super().__init__("my_backend")
+                self.api_key = api_key
+
+            async def process_startup(self):
+                self.client = MyAPIClient(self.api_key)
+
+        backend = Backend.create("my_backend", api_key="secret")
+    """
+
+    @classmethod
+    def create(cls, type_: BackendType, **kwargs) -> Backend:
+        """
+        Create a backend instance based on the backend type.
+
+        :param type_: The type of backend to create
+        :param kwargs: Additional arguments for backend initialization
+        :return: An instance of a subclass of Backend
+        :raises ValueError: If the backend type is not registered
+        """
+
+        backend = cls.get_registered_object(type_)
+
+        if backend is None:
+            raise ValueError(
+                f"Backend type '{type_}' is not registered. "
+                f"Available types: {list(cls.registry.keys()) if cls.registry else []}"
+            )
+
+        return backend(**kwargs)
+
+    def __init__(self, type_: BackendType):
+        """
+        Initialize a backend instance.
+
+        :param type_: The backend type identifier
+        """
+        self.type_ = type_
+
+    @property
+    def processes_limit(self) -> int | None:
+        """
+        :return: Maximum number of worker processes supported, None if unlimited
+        """
+        return None
+
+    @property
+    def requests_limit(self) -> int | None:
+        """
+        :return: Maximum number of concurrent requests supported globally,
+            None if unlimited
+        """
+        return None
+
+    @abstractmethod
+    async def default_model(self) -> str:
+        """
+        :return: The default model name or identifier for generation requests,
+        """
+        ...

guidellm/backends/openai.py

@@ -0,0 +1,340 @@
+"""
+OpenAI HTTP backend implementation for GuideLLM.
+
+Provides HTTP-based backend for OpenAI-compatible servers including OpenAI API,
+vLLM servers, and other compatible inference engines. Supports text and chat
+completions with streaming, authentication, and multimodal capabilities.
+Handles request formatting, response parsing, error handling, and token usage
+tracking with flexible parameter customization.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator
+from typing import Any
+
+import httpx
+
+from guidellm.backends.backend import Backend
+from guidellm.backends.response_handlers import GenerationResponseHandlerFactory
+from guidellm.schemas import GenerationRequest, GenerationResponse, RequestInfo
+
+__all__ = ["OpenAIHTTPBackend"]
+
+
+@Backend.register("openai_http")
+class OpenAIHTTPBackend(Backend):
+    """
+    HTTP backend for OpenAI-compatible servers.
+
+    Supports OpenAI API, vLLM servers, and other compatible endpoints with
+    text/chat completions, streaming, authentication, and multimodal inputs.
+    Handles request formatting, response parsing, error handling, and token
+    usage tracking with flexible parameter customization.
+
+    Example:
+    ::
+        backend = OpenAIHTTPBackend(
+            target="http://localhost:8000",
+            model="gpt-3.5-turbo",
+            api_key="your-api-key"
+        )
+
+        await backend.process_startup()
+        async for response, request_info in backend.resolve(request, info):
+            process_response(response)
+        await backend.process_shutdown()
+    """
+
+    def __init__(
+        self,
+        target: str,
+        model: str = "",
+        api_routes: dict[str, str] | None = None,
+        response_handlers: dict[str, Any] | None = None,
+        timeout: float = 60.0,
+        http2: bool = True,
+        follow_redirects: bool = True,
+        verify: bool = False,
+        validate_backend: bool | str | dict[str, Any] = True,
+    ):
+        """
+        Initialize OpenAI HTTP backend with server configuration.
+
+        :param target: Base URL of the OpenAI-compatible server
+        :param model: Model identifier for generation requests
+        :param api_routes: Custom API endpoint routes mapping
+        :param response_handlers: Custom response handlers for different request types
+        :param timeout: Request timeout in seconds
+        :param http2: Enable HTTP/2 protocol support
+        :param follow_redirects: Follow HTTP redirects automatically
+        :param verify: Enable SSL certificate verification
+        :param validate_backend: Backend validation configuration
+        """
+        super().__init__(type_="openai_http")
+
+        # Request Values
+        self.target = target.rstrip("/").removesuffix("/v1")
+        self.model = model
+
+        # Store configuration
+        self.api_routes = api_routes or {
+            "health": "health",
+            "models": "v1/models",
+            "text_completions": "v1/completions",
+            "chat_completions": "v1/chat/completions",
+            "audio_transcriptions": "v1/audio/transcriptions",
+            "audio_translations": "v1/audio/translations",
+        }
+        self.response_handlers = response_handlers
+        self.timeout = timeout
+        self.http2 = http2
+        self.follow_redirects = follow_redirects
+        self.verify = verify
+        self.validate_backend: dict[str, Any] | None = self._resolve_validate_kwargs(
+            validate_backend
+        )
+
+        # Runtime state
+        self._in_process = False
+        self._async_client: httpx.AsyncClient | None = None
+
+    @property
+    def info(self) -> dict[str, Any]:
+        """
+        Get backend configuration details.
+
+        :return: Dictionary containing backend configuration details
+        """
+        return {
+            "target": self.target,
+            "model": self.model,
+            "timeout": self.timeout,
+            "http2": self.http2,
+            "follow_redirects": self.follow_redirects,
+            "verify": self.verify,
+            "openai_paths": self.api_routes,
+            "validate_backend": self.validate_backend,
+        }
+
+    async def process_startup(self):
+        """
+        Initialize HTTP client and backend resources.
+
+        :raises RuntimeError: If backend is already initialized
+        :raises httpx.RequestError: If HTTP client cannot be created
+        """
+        if self._in_process:
+            raise RuntimeError("Backend already started up for process.")
+
+        self._async_client = httpx.AsyncClient(
+            http2=self.http2,
+            timeout=self.timeout,
+            follow_redirects=self.follow_redirects,
+            verify=self.verify,
+            # Allow unlimited connections
+            limits=httpx.Limits(
+                max_connections=None,
+                max_keepalive_connections=None,
+                keepalive_expiry=5.0,  # default
+            ),
+        )
+        self._in_process = True
+
+    async def process_shutdown(self):
+        """
+        Clean up HTTP client and backend resources.
+
+        :raises RuntimeError: If backend was not properly initialized
+        :raises httpx.RequestError: If HTTP client cannot be closed
+        """
+        if not self._in_process:
+            raise RuntimeError("Backend not started up for process.")
+
+        await self._async_client.aclose()  # type: ignore [union-attr]
+        self._async_client = None
+        self._in_process = False
+
+    async def validate(self):
+        """
+        Validate backend connectivity and configuration.
+
+        :raises RuntimeError: If backend cannot connect or validate configuration
+        """
+        if self._async_client is None:
+            raise RuntimeError("Backend not started up for process.")
+
+        if not self.validate_backend:
+            return
+
+        try:
+            response = await self._async_client.request(**self.validate_backend)
+            response.raise_for_status()
+        except Exception as exc:
+            raise RuntimeError(
+                "Backend validation request failed. Could not connect to the server "
+                "or validate the backend configuration."
+            ) from exc
+
+    async def available_models(self) -> list[str]:
+        """
+        Get available models from the target server.
+
+        :return: List of model identifiers
+        :raises httpx.HTTPError: If models endpoint returns an error
+        :raises RuntimeError: If backend is not initialized
+        """
+        if self._async_client is None:
+            raise RuntimeError("Backend not started up for process.")
+
+        target = f"{self.target}/{self.api_routes['models']}"
+        response = await self._async_client.get(target)
+        response.raise_for_status()
+
+        return [item["id"] for item in response.json()["data"]]
+
+    async def default_model(self) -> str:
+        """
+        Get the default model for this backend.
+
+        :return: Model name or None if no model is available
+        """
+        if self.model or not self._in_process:
+            return self.model
+
+        models = await self.available_models()
+        return models[0] if models else ""
+
+    async def resolve(  # type: ignore[override]
+        self,
+        request: GenerationRequest,
+        request_info: RequestInfo,
+        history: list[tuple[GenerationRequest, GenerationResponse]] | None = None,
+    ) -> AsyncIterator[tuple[GenerationResponse, RequestInfo]]:
+        """
+        Process generation request and yield progressive responses.
+
+        Handles request formatting, timing tracking, API communication, and
+        response parsing with streaming support.
+
+        :param request: Generation request with content and parameters
+        :param request_info: Request tracking info updated with timing metadata
+        :param history: Conversation history (currently not supported)
+        :raises NotImplementedError: If history is provided
+        :raises RuntimeError: If backend is not initialized
+        :raises ValueError: If request type is unsupported
+        :yields: Tuples of (response, updated_request_info) as generation progresses
+        """
+        if self._async_client is None:
+            raise RuntimeError("Backend not started up for process.")
+
+        if history is not None:
+            raise NotImplementedError("Multi-turn requests not yet supported")
+
+        if (request_path := self.api_routes.get(request.request_type)) is None:
+            raise ValueError(f"Unsupported request type '{request.request_type}'")
+
+        request_url = f"{self.target}/{request_path}"
+        request_files = (
+            {
+                key: tuple(value) if isinstance(value, list) else value
+                for key, value in request.arguments.files.items()
+            }
+            if request.arguments.files
+            else None
+        )
+        request_json = request.arguments.body if not request_files else None
+        request_data = request.arguments.body if request_files else None
+        response_handler = GenerationResponseHandlerFactory.create(
+            request.request_type, handler_overrides=self.response_handlers
+        )
+
+        if not request.arguments.stream:
+            request_info.timings.request_start = time.time()
+            response = await self._async_client.request(
+                request.arguments.method or "POST",
+                request_url,
+                params=request.arguments.params,
+                headers=request.arguments.headers,
+                json=request_json,
+                data=request_data,
+                files=request_files,
+            )
+            request_info.timings.request_end = time.time()
+            response.raise_for_status()
+            data = response.json()
+            yield response_handler.compile_non_streaming(request, data), request_info
+            return
+
+        try:
+            request_info.timings.request_start = time.time()
+
+            async with self._async_client.stream(
+                request.arguments.method or "POST",
+                request_url,
+                params=request.arguments.params,
+                headers=request.arguments.headers,
+                json=request_json,
+                data=request_data,
+                files=request_files,
+            ) as stream:
+                stream.raise_for_status()
+                end_reached = False
+
+                async for chunk in stream.aiter_lines():
+                    iter_time = time.time()
+
+                    if request_info.timings.first_request_iteration is None:
+                        request_info.timings.first_request_iteration = iter_time
+                    request_info.timings.last_request_iteration = iter_time
+                    request_info.timings.request_iterations += 1
+
+                    iterations = response_handler.add_streaming_line(chunk)
+                    if iterations is None or iterations <= 0 or end_reached:
+                        end_reached = end_reached or iterations is None
+                        continue
+
+                    if request_info.timings.first_token_iteration is None:
+                        request_info.timings.first_token_iteration = iter_time
+                        request_info.timings.token_iterations = 0
+
+                    request_info.timings.last_token_iteration = iter_time
+                    request_info.timings.token_iterations += iterations
+
+            request_info.timings.request_end = time.time()
+            yield response_handler.compile_streaming(request), request_info
+        except asyncio.CancelledError as err:
+            # Yield current result to store iterative results before propagating
+            yield response_handler.compile_streaming(request), request_info
+            raise err
+
+    def _resolve_validate_kwargs(
+        self, validate_backend: bool | str | dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if not (validate_kwargs := validate_backend):
+            return None
+
+        if validate_kwargs is True:
+            validate_kwargs = "health"
+
+        if isinstance(validate_kwargs, str) and validate_kwargs in self.api_routes:
+            validate_kwargs = f"{self.target}/{self.api_routes[validate_kwargs]}"
+
+        if isinstance(validate_kwargs, str):
+            validate_kwargs = {
+                "method": "GET",
+                "url": validate_kwargs,
+            }
+
+        if not isinstance(validate_kwargs, dict) or "url" not in validate_kwargs:
+            raise ValueError(
+                "validate_backend must be a boolean, string, or dictionary and contain "
+                f"a target URL. Got: {validate_kwargs}"
+            )
+
+        if "method" not in validate_kwargs:
+            validate_kwargs["method"] = "GET"
+
+        return validate_kwargs