guidellm 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

guidellm/__init__.py

@@ -6,14 +6,46 @@ evaluating and benchmarking large language models (LLMs).
 # flake8: noqa
 
 import os
-import transformers  # type: ignore
+import logging
+import contextlib
 
-os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Silence warnings for tokenizers
-transformers.logging.set_verbosity_error()  # Silence warnings for transformers
 
+with (
+    open(os.devnull, "w") as devnull,
+    contextlib.redirect_stderr(devnull),
+    contextlib.redirect_stdout(devnull),
+):
+    from transformers.utils import logging as hf_logging  # type: ignore[import]
 
-from .config import settings
+    # Set the log level for the transformers library to ERROR
+    # to ignore None of PyTorch, TensorFlow found
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Silence warnings for tokenizers
+    hf_logging.set_verbosity_error()
+    logging.getLogger("transformers").setLevel(logging.ERROR)
+
+from .config import (
+    settings,
+    DatasetSettings,
+    Environment,
+    LoggingSettings,
+    OpenAISettings,
+    print_config,
+    Settings,
+    reload_settings,
+)
 from .logger import configure_logger, logger
-from .main import generate_benchmark_report
 
-__all__ = ["configure_logger", "logger", "settings", "generate_benchmark_report"]
+__all__ = [
+    # Config
+    "DatasetSettings",
+    "Environment",
+    "LoggingSettings",
+    "OpenAISettings",
+    "print_config",
+    "Settings",
+    "reload_settings",
+    "settings",
+    # Logger
+    "logger",
+    "configure_logger",
+]

guidellm/__main__.py

@@ -0,0 +1,294 @@
+import asyncio
+import json
+from pathlib import Path
+from typing import get_args
+
+import click
+
+from guidellm.backend import BackendType
+from guidellm.benchmark import ProfileType, benchmark_generative_text
+from guidellm.config import print_config
+from guidellm.scheduler import StrategyType
+
+STRATEGY_PROFILE_CHOICES = set(
+    list(get_args(ProfileType)) + list(get_args(StrategyType))
+)
+
+
+def parse_json(ctx, param, value):  # noqa: ARG001
+    if value is None:
+        return None
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError as err:
+        raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err
+
+
+def parse_number_str(ctx, param, value):  # noqa: ARG001
+    if value is None:
+        return None
+
+    values = value.split(",") if "," in value else [value]
+
+    try:
+        return [int(val) if val.isdigit() else float(val) for val in values]
+    except ValueError as err:
+        raise click.BadParameter(
+            f"{param.name} must be a number or comma-separated list of numbers."
+        ) from err
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command(
+    help="Run a benchmark against a generative model using the specified arguments."
+)
+@click.option(
+    "--target",
+    required=True,
+    type=str,
+    help="The target path for the backend to run benchmarks against. For example, http://localhost:8000",
+)
+@click.option(
+    "--backend-type",
+    type=click.Choice(list(get_args(BackendType))),
+    help=(
+        "The type of backend to use to run requests against. Defaults to 'openai_http'."
+        f" Supported types: {', '.join(get_args(BackendType))}"
+    ),
+    default="openai_http",
+)
+@click.option(
+    "--backend-args",
+    callback=parse_json,
+    default=None,
+    help=(
+        "A JSON string containing any arguments to pass to the backend as a "
+        "dict with **kwargs."
+    ),
+)
+@click.option(
+    "--model",
+    default=None,
+    type=str,
+    help=(
+        "The ID of the model to benchmark within the backend. "
+        "If None provided (default), then it will use the first model available."
+    ),
+)
+@click.option(
+    "--processor",
+    default=None,
+    type=str,
+    help=(
+        "The processor or tokenizer to use to calculate token counts for statistics "
+        "and synthetic data generation. If None provided (default), will load "
+        "using the model arg, if needed."
+    ),
+)
+@click.option(
+    "--processor-args",
+    default=None,
+    callback=parse_json,
+    help=(
+        "A JSON string containing any arguments to pass to the processor constructor "
+        "as a dict with **kwargs."
+    ),
+)
+@click.option(
+    "--data",
+    required=True,
+    type=str,
+    help=(
+        "The HuggingFace dataset ID, a path to a HuggingFace dataset, "
+        "a path to a data file csv, json, jsonl, or txt, "
+        "or a synthetic data config as a json or key=value string."
+    ),
+)
+@click.option(
+    "--data-args",
+    callback=parse_json,
+    help=(
+        "A JSON string containing any arguments to pass to the dataset creation "
+        "as a dict with **kwargs."
+    ),
+)
+@click.option(
+    "--data-sampler",
+    default=None,
+    type=click.Choice(["random"]),
+    help=(
+        "The data sampler type to use. 'random' will add a random shuffle on the data. "
+        "Defaults to None"
+    ),
+)
+@click.option(
+    "--rate-type",
+    required=True,
+    type=click.Choice(STRATEGY_PROFILE_CHOICES),
+    help=(
+        "The type of benchmark to run. "
+        f"Supported types {', '.join(STRATEGY_PROFILE_CHOICES)}. "
+    ),
+)
+@click.option(
+    "--rate",
+    default=None,
+    callback=parse_number_str,
+    help=(
+        "The rates to run the benchmark at. "
+        "Can be a single number or a comma-separated list of numbers. "
+        "For rate-type=sweep, this is the number of benchmarks it runs in the sweep. "
+        "For rate-type=concurrent, this is the number of concurrent requests. "
+        "For rate-type=async,constant,poisson, this is the rate requests per second. "
+        "For rate-type=synchronous,throughput, this must not be set."
+    ),
+)
+@click.option(
+    "--max-seconds",
+    type=float,
+    help=(
+        "The maximum number of seconds each benchmark can run for. "
+        "If None, will run until max_requests or the data is exhausted."
+    ),
+)
+@click.option(
+    "--max-requests",
+    type=int,
+    help=(
+        "The maximum number of requests each benchmark can run for. "
+        "If None, will run until max_seconds or the data is exhausted."
+    ),
+)
+@click.option(
+    "--warmup-percent",
+    type=float,
+    default=None,
+    help=(
+        "The percent of the benchmark (based on max-seconds, max-requets, "
+        "or lenth of dataset) to run as a warmup and not include in the final results. "
+        "Defaults to None."
+    ),
+)
+@click.option(
+    "--cooldown-percent",
+    type=float,
+    help=(
+        "The percent of the benchmark (based on max-seconds, max-requets, or lenth "
+        "of dataset) to run as a cooldown and not include in the final results. "
+        "Defaults to None."
+    ),
+)
+@click.option(
+    "--disable-progress",
+    is_flag=True,
+    help="Set this flag to disable progress updates to the console",
+)
+@click.option(
+    "--display-scheduler-stats",
+    is_flag=True,
+    help="Set this flag to display stats for the processes running the benchmarks",
+)
+@click.option(
+    "--disable-console-outputs",
+    is_flag=True,
+    help="Set this flag to disable console output",
+)
+@click.option(
+    "--output-path",
+    type=click.Path(),
+    default=Path.cwd() / "benchmarks.json",
+    help=(
+        "The path to save the output to. If it is a directory, "
+        "it will save benchmarks.json under it. "
+        "Otherwise, json, yaml, or csv files are supported for output types "
+        "which will be read from the extension for the file path."
+    ),
+)
+@click.option(
+    "--output-extras",
+    callback=parse_json,
+    help="A JSON string of extra data to save with the output benchmarks",
+)
+@click.option(
+    "--output-sampling",
+    type=int,
+    help=(
+        "The number of samples to save in the output file. "
+        "If None (default), will save all samples."
+    ),
+    default=None,
+)
+@click.option(
+    "--random-seed",
+    default=42,
+    type=int,
+    help="The random seed to use for benchmarking to ensure reproducibility.",
+)
+def benchmark(
+    target,
+    backend_type,
+    backend_args,
+    model,
+    processor,
+    processor_args,
+    data,
+    data_args,
+    data_sampler,
+    rate_type,
+    rate,
+    max_seconds,
+    max_requests,
+    warmup_percent,
+    cooldown_percent,
+    disable_progress,
+    display_scheduler_stats,
+    disable_console_outputs,
+    output_path,
+    output_extras,
+    output_sampling,
+    random_seed,
+):
+    asyncio.run(
+        benchmark_generative_text(
+            target=target,
+            backend_type=backend_type,
+            backend_args=backend_args,
+            model=model,
+            processor=processor,
+            processor_args=processor_args,
+            data=data,
+            data_args=data_args,
+            data_sampler=data_sampler,
+            rate_type=rate_type,
+            rate=rate,
+            max_seconds=max_seconds,
+            max_requests=max_requests,
+            warmup_percent=warmup_percent,
+            cooldown_percent=cooldown_percent,
+            show_progress=not disable_progress,
+            show_progress_scheduler_stats=display_scheduler_stats,
+            output_console=not disable_console_outputs,
+            output_path=output_path,
+            output_extras=output_extras,
+            output_sampling=output_sampling,
+            random_seed=random_seed,
+        )
+    )
+
+
+@cli.command(
+    help=(
+        "Print out the available configuration settings that can be set "
+        "through environment variables."
+    )
+)
+def config():
+    print_config()
+
+
+if __name__ == "__main__":
+    cli()

guidellm/backend/__init__.py

@@ -1,10 +1,23 @@
-from .base import Backend, BackendEngine, BackendEnginePublic, GenerativeResponse
-from .openai import OpenAIBackend
+from .backend import (
+    Backend,
+    BackendType,
+)
+from .openai import CHAT_COMPLETIONS_PATH, TEXT_COMPLETIONS_PATH, OpenAIHTTPBackend
+from .response import (
+    RequestArgs,
+    ResponseSummary,
+    StreamingResponseType,
+    StreamingTextResponse,
+)
 
 __all__ = [
+    "StreamingResponseType",
+    "StreamingTextResponse",
+    "RequestArgs",
+    "ResponseSummary",
     "Backend",
-    "BackendEngine",
-    "BackendEnginePublic",
-    "GenerativeResponse",
-    "OpenAIBackend",
+    "BackendType",
+    "OpenAIHTTPBackend",
+    "TEXT_COMPLETIONS_PATH",
+    "CHAT_COMPLETIONS_PATH",
 ]

guidellm/backend/backend.py

@@ -0,0 +1,238 @@
+from abc import ABC, abstractmethod
+from collections.abc import AsyncGenerator
+from pathlib import Path
+from typing import Any, Literal, Optional, Union
+
+from loguru import logger
+from PIL import Image
+
+from guidellm.backend.response import ResponseSummary, StreamingTextResponse
+
+__all__ = [
+    "Backend",
+    "BackendType",
+]
+
+
+BackendType = Literal["openai_http"]
+
+
+class Backend(ABC):
+    """
+    Abstract base class for generative AI backends.
+
+    This class provides a common interface for creating and interacting with different
+    generative AI backends. Subclasses should implement the abstract methods to
+    define specific backend behavior.
+
+    :cvar _registry: A registration dictionary that maps BackendType to backend classes.
+    :param type_: The type of the backend.
+    """
+
+    _registry: dict[BackendType, "type[Backend]"] = {}
+
+    @classmethod
+    def register(cls, backend_type: BackendType):
+        """
+        A decorator to register a backend class in the backend registry.
+
+        :param backend_type: The type of backend to register.
+        :type backend_type: BackendType
+        :return: The decorated backend class.
+        :rtype: Type[Backend]
+        """
+        if backend_type in cls._registry:
+            raise ValueError(f"Backend type already registered: {backend_type}")
+
+        if not issubclass(cls, Backend):
+            raise TypeError("Only subclasses of Backend can be registered")
+
+        def inner_wrapper(wrapped_class: type["Backend"]):
+            cls._registry[backend_type] = wrapped_class
+            logger.info("Registered backend type: {}", backend_type)
+            return wrapped_class
+
+        return inner_wrapper
+
+    @classmethod
+    def create(cls, type_: BackendType, **kwargs) -> "Backend":
+        """
+        Factory method to create a backend instance based on the backend type.
+
+        :param type_: The type of backend to create.
+        :type type_: BackendType
+        :param kwargs: Additional arguments for backend initialization.
+        :return: An instance of a subclass of Backend.
+        :rtype: Backend
+        :raises ValueError: If the backend type is not registered.
+        """
+
+        logger.info("Creating backend of type {}", type_)
+
+        if type_ not in cls._registry:
+            err = ValueError(f"Unsupported backend type: {type_}")
+            logger.error("{}", err)
+            raise err
+
+        return Backend._registry[type_](**kwargs)
+
+    def __init__(self, type_: BackendType):
+        self._type = type_
+
+    @property
+    def type_(self) -> BackendType:
+        """
+        :return: The type of the backend.
+        """
+        return self._type
+
+    @property
+    @abstractmethod
+    def target(self) -> str:
+        """
+        :return: The target location for the backend.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def model(self) -> Optional[str]:
+        """
+        :return: The model used for the backend requests.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def info(self) -> dict[str, Any]:
+        """
+        :return: The information about the backend.
+        """
+        ...
+
+    async def validate(self):
+        """
+        Handle final setup and validate the backend is ready for use.
+        If not successful, raises the appropriate exception.
+        """
+        logger.info("{} validating backend {}", self.__class__.__name__, self.type_)
+        await self.check_setup()
+        models = await self.available_models()
+        if not models:
+            raise ValueError("No models available for the backend")
+
+        async for _ in self.text_completions(
+            prompt="Test connection", output_token_count=1
+        ):  # type: ignore[attr-defined]
+            pass
+
+    @abstractmethod
+    async def check_setup(self):
+        """
+        Check the setup for the backend.
+        If unsuccessful, raises the appropriate exception.
+
+        :raises ValueError: If the setup check fails.
+        """
+        ...
+
+    @abstractmethod
+    async def prepare_multiprocessing(self):
+        """
+        Prepare the backend for use in a multiprocessing environment.
+        This is useful for backends that have instance state that can not
+        be shared across processes and should be cleared out and re-initialized
+        for each new process.
+        """
+        ...
+
+    @abstractmethod
+    async def available_models(self) -> list[str]:
+        """
+        Get the list of available models for the backend.
+
+        :return: The list of available models.
+        :rtype: List[str]
+        """
+        ...
+
+    @abstractmethod
+    async def text_completions(
+        self,
+        prompt: Union[str, list[str]],
+        request_id: Optional[str] = None,
+        prompt_token_count: Optional[int] = None,
+        output_token_count: Optional[int] = None,
+        **kwargs,
+    ) -> AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None]:
+        """
+        Generate text only completions for the given prompt.
+        Does not support multiple modalities, complicated chat interfaces,
+        or chat templates. Specifically, it requests with only the prompt.
+
+        :param prompt: The prompt (or list of prompts) to generate a completion for.
+            If a list is supplied, these are concatenated and run through the model
+            for a single prompt.
+        :param request_id: The unique identifier for the request, if any.
+            Added to logging statements and the response for tracking purposes.
+        :param prompt_token_count: The number of tokens measured in the prompt, if any.
+            Returned in the response stats for later analysis, if applicable.
+        :param output_token_count: If supplied, the number of tokens to enforce
+            generation of for the output for this request.
+        :param kwargs: Additional keyword arguments to pass with the request.
+        :return: An async generator that yields a StreamingTextResponse for start,
+            a StreamingTextResponse for each received iteration,
+            and a ResponseSummary for the final response.
+        """
+        ...
+
+    @abstractmethod
+    async def chat_completions(
+        self,
+        content: Union[
+            str,
+            list[Union[str, dict[str, Union[str, dict[str, str]]], Path, Image.Image]],
+            Any,
+        ],
+        request_id: Optional[str] = None,
+        prompt_token_count: Optional[int] = None,
+        output_token_count: Optional[int] = None,
+        raw_content: bool = False,
+        **kwargs,
+    ) -> AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None]:
+        """
+        Generate chat completions for the given content.
+        Supports multiple modalities, complicated chat interfaces, and chat templates.
+        Specifically, it requests with the content, which can be any combination of
+        text, images, and audio provided the target model supports it,
+        and returns the output text. Additionally, any chat templates
+        for the model are applied within the backend.
+
+        :param content: The content (or list of content) to generate a completion for.
+            This supports any combination of text, images, and audio (model dependent).
+            Supported text only request examples:
+                content="Sample prompt", content=["Sample prompt", "Second prompt"],
+                content=[{"type": "text", "value": "Sample prompt"}.
+            Supported text and image request examples:
+                content=["Describe the image", PIL.Image.open("image.jpg")],
+                content=["Describe the image", Path("image.jpg")],
+                content=["Describe the image", {"type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}].
+            Supported text and audio request examples:
+                content=["Transcribe the audio", Path("audio.wav")],
+                content=["Transcribe the audio", {"type": "input_audio",
+                "input_audio": {"data": f"{base64_bytes}", "format": "wav}].
+            Additionally, if raw_content=True then the content is passed directly to the
+            backend without any processing.
+        :param request_id: The unique identifier for the request, if any.
+            Added to logging statements and the response for tracking purposes.
+        :param prompt_token_count: The number of tokens measured in the prompt, if any.
+            Returned in the response stats for later analysis, if applicable.
+        :param output_token_count: If supplied, the number of tokens to enforce
+            generation of for the output for this request.
+        :param kwargs: Additional keyword arguments to pass with the request.
+        :return: An async generator that yields a StreamingTextResponse for start,
+            a StreamingTextResponse for each received iteration,
+            and a ResponseSummary for the final response.
+        """
+        ...