guidellm 0.3.0rc20250507__py3-none-any.whl → 0.4.0a2__py3-none-any.whl

guidellm/__init__.py

@@ -3,15 +3,12 @@ Guidellm is a package that provides an easy and intuitive interface for
 evaluating and benchmarking large language models (LLMs).
 """
 
-# flake8: noqa
-
-import os
-import logging
 import contextlib
-
+import logging
+import os
 
 with (
-    open(os.devnull, "w") as devnull,
+    open(os.devnull, "w") as devnull,  # noqa: PTH123
     contextlib.redirect_stderr(devnull),
     contextlib.redirect_stdout(devnull),
 ):
@@ -24,28 +21,26 @@ with (
     logging.getLogger("transformers").setLevel(logging.ERROR)
 
 from .config import (
-    settings,
     DatasetSettings,
     Environment,
     LoggingSettings,
     OpenAISettings,
-    print_config,
     Settings,
+    print_config,
     reload_settings,
+    settings,
 )
 from .logger import configure_logger, logger
 
 __all__ = [
-    # Config
     "DatasetSettings",
     "Environment",
     "LoggingSettings",
     "OpenAISettings",
-    "print_config",
     "Settings",
+    "configure_logger",
+    "logger",
+    "print_config",
     "reload_settings",
     "settings",
-    # Logger
-    "logger",
-    "configure_logger",
 ]

guidellm/__main__.py

@@ -1,54 +1,70 @@
 import asyncio
-import json
+import codecs
 from pathlib import Path
 from typing import get_args
 
 import click
+from pydantic import ValidationError
 
 from guidellm.backend import BackendType
-from guidellm.benchmark import ProfileType, benchmark_generative_text
+from guidellm.benchmark import (
+    ProfileType,
+    reimport_benchmarks_report,
+)
+from guidellm.benchmark.entrypoints import benchmark_with_scenario
+from guidellm.benchmark.scenario import GenerativeTextScenario, get_builtin_scenarios
 from guidellm.config import print_config
+from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset
 from guidellm.scheduler import StrategyType
+from guidellm.utils import DefaultGroupHandler
+from guidellm.utils import cli as cli_tools
 
-STRATEGY_PROFILE_CHOICES = set(
-    list(get_args(ProfileType)) + list(get_args(StrategyType))
+STRATEGY_PROFILE_CHOICES = list(
+    set(list(get_args(ProfileType)) + list(get_args(StrategyType)))
 )
 
 
-def parse_json(ctx, param, value):  # noqa: ARG001
-    if value is None:
-        return None
-    try:
-        return json.loads(value)
-    except json.JSONDecodeError as err:
-        raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err
-
-
-def parse_number_str(ctx, param, value):  # noqa: ARG001
-    if value is None:
-        return None
-
-    values = value.split(",") if "," in value else [value]
-
-    try:
-        return [int(val) if val.isdigit() else float(val) for val in values]
-    except ValueError as err:
-        raise click.BadParameter(
-            f"{param.name} must be a number or comma-separated list of numbers."
-        ) from err
-
-
 @click.group()
+@click.version_option(package_name="guidellm", message="guidellm version: %(version)s")
 def cli():
     pass
 
 
-@cli.command(
-    help="Run a benchmark against a generative model using the specified arguments."
+@cli.group(
+    help="Commands to run a new benchmark or load a prior one.",
+    cls=DefaultGroupHandler,
+    default="run",
+)
+def benchmark():
+    pass
+
+
+@benchmark.command(
+    "run",
+    help="Run a benchmark against a generative model using the specified arguments.",
+    context_settings={"auto_envvar_prefix": "GUIDELLM"},
+)
+@click.option(
+    "--scenario",
+    type=cli_tools.Union(
+        click.Path(
+            exists=True,
+            readable=True,
+            file_okay=True,
+            dir_okay=False,
+            path_type=Path,
+        ),
+        click.Choice(get_builtin_scenarios()),
+    ),
+    default=None,
+    help=(
+        "The name of a builtin scenario or path to a config file. "
+        "Missing values from the config will use defaults. "
+        "Options specified on the commandline will override the scenario."
+    ),
 )
 @click.option(
     "--target",
-    required=True,
     type=str,
     help="The target path for the backend to run benchmarks against. For example, http://localhost:8000",
 )
@@ -59,20 +75,22 @@ def cli():
         "The type of backend to use to run requests against. Defaults to 'openai_http'."
         f" Supported types: {', '.join(get_args(BackendType))}"
     ),
-    default="openai_http",
+    default=GenerativeTextScenario.get_default("backend_type"),
 )
 @click.option(
     "--backend-args",
-    callback=parse_json,
-    default=None,
+    callback=cli_tools.parse_json,
+    default=GenerativeTextScenario.get_default("backend_args"),
     help=(
         "A JSON string containing any arguments to pass to the backend as a "
-        "dict with **kwargs."
+        "dict with **kwargs. Headers can be removed by setting their value to "
+        "null. For example: "
+        """'{"headers": {"Authorization": null, "Custom-Header": "Custom-Value"}}'"""
     ),
 )
 @click.option(
     "--model",
-    default=None,
+    default=GenerativeTextScenario.get_default("model"),
     type=str,
     help=(
         "The ID of the model to benchmark within the backend. "
@@ -81,7 +99,7 @@ def cli():
 )
 @click.option(
     "--processor",
-    default=None,
+    default=GenerativeTextScenario.get_default("processor"),
     type=str,
     help=(
         "The processor or tokenizer to use to calculate token counts for statistics "
@@ -91,8 +109,8 @@ def cli():
 )
 @click.option(
     "--processor-args",
-    default=None,
-    callback=parse_json,
+    default=GenerativeTextScenario.get_default("processor_args"),
+    callback=cli_tools.parse_json,
     help=(
         "A JSON string containing any arguments to pass to the processor constructor "
         "as a dict with **kwargs."
@@ -100,7 +118,6 @@ def cli():
 )
 @click.option(
     "--data",
-    required=True,
     type=str,
     help=(
         "The HuggingFace dataset ID, a path to a HuggingFace dataset, "
@@ -110,7 +127,8 @@ def cli():
 )
 @click.option(
     "--data-args",
-    callback=parse_json,
+    default=GenerativeTextScenario.get_default("data_args"),
+    callback=cli_tools.parse_json,
     help=(
         "A JSON string containing any arguments to pass to the dataset creation "
         "as a dict with **kwargs."
@@ -118,7 +136,7 @@ def cli():
 )
 @click.option(
     "--data-sampler",
-    default=None,
+    default=GenerativeTextScenario.get_default("data_sampler"),
     type=click.Choice(["random"]),
     help=(
         "The data sampler type to use. 'random' will add a random shuffle on the data. "
@@ -127,7 +145,6 @@ def cli():
 )
 @click.option(
     "--rate-type",
-    required=True,
     type=click.Choice(STRATEGY_PROFILE_CHOICES),
     help=(
         "The type of benchmark to run. "
@@ -136,8 +153,7 @@ def cli():
 )
 @click.option(
     "--rate",
-    default=None,
-    callback=parse_number_str,
+    default=GenerativeTextScenario.get_default("rate"),
     help=(
         "The rates to run the benchmark at. "
         "Can be a single number or a comma-separated list of numbers. "
@@ -150,6 +166,7 @@ def cli():
 @click.option(
     "--max-seconds",
     type=float,
+    default=GenerativeTextScenario.get_default("max_seconds"),
     help=(
         "The maximum number of seconds each benchmark can run for. "
         "If None, will run until max_requests or the data is exhausted."
@@ -158,6 +175,7 @@ def cli():
 @click.option(
     "--max-requests",
     type=int,
+    default=GenerativeTextScenario.get_default("max_requests"),
     help=(
         "The maximum number of requests each benchmark can run for. "
         "If None, will run until max_seconds or the data is exhausted."
@@ -166,7 +184,7 @@ def cli():
 @click.option(
     "--warmup-percent",
     type=float,
-    default=None,
+    default=GenerativeTextScenario.get_default("warmup_percent"),
     help=(
         "The percent of the benchmark (based on max-seconds, max-requets, "
         "or lenth of dataset) to run as a warmup and not include in the final results. "
@@ -176,6 +194,7 @@ def cli():
 @click.option(
     "--cooldown-percent",
     type=float,
+    default=GenerativeTextScenario.get_default("cooldown_percent"),
     help=(
         "The percent of the benchmark (based on max-seconds, max-requets, or lenth "
         "of dataset) to run as a cooldown and not include in the final results. "
@@ -204,13 +223,13 @@ def cli():
     help=(
         "The path to save the output to. If it is a directory, "
         "it will save benchmarks.json under it. "
-        "Otherwise, json, yaml, or csv files are supported for output types "
+        "Otherwise, json, yaml, csv, or html files are supported for output types "
         "which will be read from the extension for the file path."
     ),
 )
 @click.option(
     "--output-extras",
-    callback=parse_json,
+    callback=cli_tools.parse_json,
     help="A JSON string of extra data to save with the output benchmarks",
 )
 @click.option(
@@ -220,15 +239,16 @@ def cli():
         "The number of samples to save in the output file. "
         "If None (default), will save all samples."
     ),
-    default=None,
+    default=GenerativeTextScenario.get_default("output_sampling"),
 )
 @click.option(
     "--random-seed",
-    default=42,
+    default=GenerativeTextScenario.get_default("random_seed"),
     type=int,
     help="The random seed to use for benchmarking to ensure reproducibility.",
 )
-def benchmark(
+def run(
+    scenario,
     target,
     backend_type,
     backend_args,
@@ -252,43 +272,244 @@ def benchmark(
     output_sampling,
     random_seed,
 ):
+    click_ctx = click.get_current_context()
+
+    overrides = cli_tools.set_if_not_default(
+        click_ctx,
+        target=target,
+        backend_type=backend_type,
+        backend_args=backend_args,
+        model=model,
+        processor=processor,
+        processor_args=processor_args,
+        data=data,
+        data_args=data_args,
+        data_sampler=data_sampler,
+        rate_type=rate_type,
+        rate=rate,
+        max_seconds=max_seconds,
+        max_requests=max_requests,
+        warmup_percent=warmup_percent,
+        cooldown_percent=cooldown_percent,
+        output_sampling=output_sampling,
+        random_seed=random_seed,
+    )
+
+    try:
+        # If a scenario file was specified read from it
+        if scenario is None:
+            _scenario = GenerativeTextScenario.model_validate(overrides)
+        elif isinstance(scenario, Path):
+            _scenario = GenerativeTextScenario.from_file(scenario, overrides)
+        else:  # Only builtins can make it here; click will catch anything else
+            _scenario = GenerativeTextScenario.from_builtin(scenario, overrides)
+    except ValidationError as e:
+        # Translate pydantic valdation error to click argument error
+        errs = e.errors(include_url=False, include_context=True, include_input=True)
+        param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-")
+        raise click.BadParameter(
+            errs[0]["msg"], ctx=click_ctx, param_hint=param_name
+        ) from e
+
     asyncio.run(
-        benchmark_generative_text(
-            target=target,
-            backend_type=backend_type,
-            backend_args=backend_args,
-            model=model,
-            processor=processor,
-            processor_args=processor_args,
-            data=data,
-            data_args=data_args,
-            data_sampler=data_sampler,
-            rate_type=rate_type,
-            rate=rate,
-            max_seconds=max_seconds,
-            max_requests=max_requests,
-            warmup_percent=warmup_percent,
-            cooldown_percent=cooldown_percent,
+        benchmark_with_scenario(
+            scenario=_scenario,
             show_progress=not disable_progress,
             show_progress_scheduler_stats=display_scheduler_stats,
             output_console=not disable_console_outputs,
             output_path=output_path,
             output_extras=output_extras,
-            output_sampling=output_sampling,
-            random_seed=random_seed,
         )
     )
 
 
+@benchmark.command("from-file", help="Load a saved benchmark report.")
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, exists=True),
+    default=Path.cwd() / "benchmarks.json",
+)
+@click.option(
+    "--output-path",
+    type=click.Path(file_okay=True, dir_okay=True, exists=False),
+    default=None,
+    is_flag=False,
+    flag_value=Path.cwd() / "benchmarks_reexported.json",
+    help=(
+        "Allows re-exporting the benchmarks to another format. "
+        "The path to save the output to. If it is a directory, "
+        "it will save benchmarks.json under it. "
+        "Otherwise, json, yaml, or csv files are supported for output types "
+        "which will be read from the extension for the file path. "
+        "This input is optional. If the output path flag is not provided, "
+        "the benchmarks will not be reexported. If the flag is present but "
+        "no value is specified, it will default to the current directory "
+        "with the file name `benchmarks_reexported.json`."
+    ),
+)
+def from_file(path, output_path):
+    reimport_benchmarks_report(path, output_path)
+
+
+def decode_escaped_str(_ctx, _param, value):
+    """
+    Click auto adds characters. For example, when using --pad-char "\n",
+    it parses it as "\\n". This method decodes the string to handle escape
+    sequences correctly.
+    """
+    if value is None:
+        return None
+    try:
+        return codecs.decode(value, "unicode_escape")
+    except Exception as e:
+        raise click.BadParameter(f"Could not decode escape sequences: {e}") from e
+
+
 @cli.command(
+    short_help="Prints environment variable settings.",
     help=(
         "Print out the available configuration settings that can be set "
         "through environment variables."
-    )
+    ),
 )
 def config():
     print_config()
 
 
+@cli.group(help="General preprocessing tools and utilities.")
+def preprocess():
+    pass
+
+
+@preprocess.command(
+    help=(
+        "Convert a dataset to have specific prompt and output token sizes.\n"
+        "DATA: Path to the input dataset or dataset ID.\n"
+        "OUTPUT_PATH: Path to save the converted dataset, including file suffix."
+    ),
+    context_settings={"auto_envvar_prefix": "GUIDELLM"},
+)
+@click.argument(
+    "data",
+    type=str,
+    required=True,
+)
+@click.argument(
+    "output_path",
+    type=click.Path(file_okay=True, dir_okay=False, writable=True, resolve_path=True),
+    required=True,
+)
+@click.option(
+    "--processor",
+    type=str,
+    required=True,
+    help=(
+        "The processor or tokenizer to use to calculate token counts for statistics "
+        "and synthetic data generation."
+    ),
+)
+@click.option(
+    "--processor-args",
+    default=None,
+    callback=cli_tools.parse_json,
+    help=(
+        "A JSON string containing any arguments to pass to the processor constructor "
+        "as a dict with **kwargs."
+    ),
+)
+@click.option(
+    "--data-args",
+    callback=cli_tools.parse_json,
+    help=(
+        "A JSON string containing any arguments to pass to the dataset creation "
+        "as a dict with **kwargs."
+    ),
+)
+@click.option(
+    "--short-prompt-strategy",
+    type=click.Choice([s.value for s in ShortPromptStrategy]),
+    default=ShortPromptStrategy.IGNORE.value,
+    show_default=True,
+    help="Strategy to handle prompts shorter than the target length. ",
+)
+@click.option(
+    "--pad-char",
+    type=str,
+    default="",
+    callback=decode_escaped_str,
+    help="The token to pad short prompts with when using the 'pad' strategy.",
+)
+@click.option(
+    "--concat-delimiter",
+    type=str,
+    default="",
+    help=(
+        "The delimiter to use when concatenating prompts that are too short."
+        " Used when strategy is 'concatenate'."
+    ),
+)
+@click.option(
+    "--prompt-tokens",
+    type=str,
+    default=None,
+    help="Prompt tokens config (JSON, YAML file or key=value string)",
+)
+@click.option(
+    "--output-tokens",
+    type=str,
+    default=None,
+    help="Output tokens config (JSON, YAML file or key=value string)",
+)
+@click.option(
+    "--push-to-hub",
+    is_flag=True,
+    help="Set this flag to push the converted dataset to the Hugging Face Hub.",
+)
+@click.option(
+    "--hub-dataset-id",
+    type=str,
+    default=None,
+    help="The Hugging Face Hub dataset ID to push to. "
+    "Required if --push-to-hub is used.",
+)
+@click.option(
+    "--random-seed",
+    type=int,
+    default=42,
+    show_default=True,
+    help="Random seed for prompt token sampling and output tokens sampling.",
+)
+def dataset(
+    data,
+    output_path,
+    processor,
+    processor_args,
+    data_args,
+    short_prompt_strategy,
+    pad_char,
+    concat_delimiter,
+    prompt_tokens,
+    output_tokens,
+    push_to_hub,
+    hub_dataset_id,
+    random_seed,
+):
+    process_dataset(
+        data=data,
+        output_path=output_path,
+        processor=processor,
+        prompt_tokens=prompt_tokens,
+        output_tokens=output_tokens,
+        processor_args=processor_args,
+        data_args=data_args,
+        short_prompt_strategy=short_prompt_strategy,
+        pad_char=pad_char,
+        concat_delimiter=concat_delimiter,
+        push_to_hub=push_to_hub,
+        hub_dataset_id=hub_dataset_id,
+        random_seed=random_seed,
+    )
+
+
 if __name__ == "__main__":
     cli()

guidellm/backend/__init__.py

@@ -11,13 +11,13 @@ from .response import (
 )
 
 __all__ = [
-    "StreamingResponseType",
-    "StreamingTextResponse",
-    "RequestArgs",
-    "ResponseSummary",
+    "CHAT_COMPLETIONS_PATH",
+    "TEXT_COMPLETIONS_PATH",
     "Backend",
     "BackendType",
     "OpenAIHTTPBackend",
-    "TEXT_COMPLETIONS_PATH",
-    "CHAT_COMPLETIONS_PATH",
+    "RequestArgs",
+    "ResponseSummary",
+    "StreamingResponseType",
+    "StreamingTextResponse",
 ]

guidellm/backend/backend.py

@@ -7,6 +7,7 @@ from loguru import logger
 from PIL import Image
 
 from guidellm.backend.response import ResponseSummary, StreamingTextResponse
+from guidellm.config import settings
 
 __all__ = [
     "Backend",
@@ -110,6 +111,14 @@ class Backend(ABC):
         """
         ...
 
+    @abstractmethod
+    async def reset(self) -> None:
+        """
+        Reset the connection object. This is useful for backends that
+        reuse connections or have state that needs to be cleared.
+        """
+        ...
+
     async def validate(self):
         """
         Handle final setup and validate the backend is ready for use.
@@ -121,10 +130,22 @@ class Backend(ABC):
         if not models:
             raise ValueError("No models available for the backend")
 
-        async for _ in self.text_completions(
-            prompt="Test connection", output_token_count=1
-        ):  # type: ignore[attr-defined]
-            pass
+        # Use the preferred route defined in the global settings when performing the
+        # validation request. This avoids calling an unavailable endpoint (ie
+        # /v1/completions) when the deployment only supports the chat completions
+        # endpoint.
+        if settings.preferred_route == "chat_completions":
+            async for _ in self.chat_completions(  # type: ignore[attr-defined]
+                content="Test connection", output_token_count=1
+            ):
+                pass
+        else:
+            async for _ in self.text_completions(  # type: ignore[attr-defined]
+                prompt="Test connection", output_token_count=1
+            ):
+                pass
+
+        await self.reset()
 
     @abstractmethod
     async def check_setup(self):