guidellm 0.3.0rc20250507__py3-none-any.whl → 0.3.1__py3-none-any.whl

guidellm/benchmark/output.py

@@ -2,6 +2,7 @@ import csv
 import json
 import math
 from collections import OrderedDict
+from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Literal, Optional, Union
@@ -25,12 +26,16 @@ from guidellm.objects import (
     StandardBaseModel,
     StatusDistributionSummary,
 )
+from guidellm.presentation import UIDataBuilder
+from guidellm.presentation.injector import create_report
 from guidellm.scheduler import strategy_display_str
 from guidellm.utils import Colors, split_text_list_by_length
+from guidellm.utils.dict import recursive_key_update
+from guidellm.utils.text import camelize_str
 
 __all__ = [
-    "GenerativeBenchmarksReport",
     "GenerativeBenchmarksConsole",
+    "GenerativeBenchmarksReport",
 ]
 
 
@@ -68,6 +73,9 @@ class GenerativeBenchmarksReport(StandardBaseModel):
         if type_ == "csv":
             raise ValueError(f"CSV file type is not supported for loading: {path}.")
 
+        if type_ == "html":
+            raise ValueError(f"HTML file type is not supported for loading: {path}.")
+
         raise ValueError(f"Unsupported file type: {type_} for {path}.")
 
     benchmarks: list[GenerativeBenchmark] = Field(
@@ -114,6 +122,9 @@ class GenerativeBenchmarksReport(StandardBaseModel):
         if type_ == "csv":
             return self.save_csv(path)
 
+        if type_ == "html":
+            return self.save_html(path)
+
         raise ValueError(f"Unsupported file type: {type_} for {path}.")
 
     def save_json(self, path: Union[str, Path]) -> Path:
@@ -133,10 +144,9 @@ class GenerativeBenchmarksReport(StandardBaseModel):
             )
 
         model_dict = self.model_dump()
-        model_json = json.dumps(model_dict)
 
-        with path.open("w") as file:
-            file.write(model_json)
+        with path.open("w", encoding="utf-8") as file:
+            json.dump(model_dict, file, ensure_ascii=False, indent=4)
 
         return path
 
@@ -220,11 +230,29 @@ class GenerativeBenchmarksReport(StandardBaseModel):
 
         return path
 
+    def save_html(self, path: Union[str, Path]) -> Path:
+        """
+        Download html, inject report data and save to a file.
+
+        :param path: The path to create the report at.
+        :return: The path to the report.
+        """
+
+        data_builder = UIDataBuilder(self.benchmarks)
+        data = data_builder.to_dict()
+        camel_data = recursive_key_update(deepcopy(data), camelize_str)
+        ui_api_data = {}
+        for k, v in camel_data.items():
+            key = f"window.{k} = {{}};"
+            value = f"window.{k} = {json.dumps(v, indent=2)};\n"
+            ui_api_data[key] = value
+        return create_report(ui_api_data, path)
+
     @staticmethod
     def _file_setup(
         path: Union[str, Path],
-        default_file_type: Literal["json", "yaml", "csv"] = "json",
-    ) -> tuple[Path, Literal["json", "yaml", "csv"]]:
+        default_file_type: Literal["json", "yaml", "csv", "html"] = "json",
+    ) -> tuple[Path, Literal["json", "yaml", "csv", "html"]]:
         path = Path(path) if not isinstance(path, Path) else path
 
         if path.is_dir():
@@ -242,7 +270,13 @@ class GenerativeBenchmarksReport(StandardBaseModel):
         if path_suffix in [".csv"]:
             return path, "csv"
 
-        raise ValueError(f"Unsupported file extension: {path_suffix} for {path}.")
+        if path_suffix in [".html"]:
+            return path, "html"
+
+        raise ValueError(
+            f"Unsupported file extension: {path_suffix} for {path}; "
+            "expected json, yaml, csv, or html."
+        )
 
     @staticmethod
     def _benchmark_desc_headers_and_values(
@@ -889,7 +923,7 @@ class GenerativeBenchmarksConsole:
             "Request Stats": (1, 2),
             "Out Tok/sec": (3, 3),
             "Tot Tok/sec": (4, 4),
-            "Req Latency (ms)": (5, 7),
+            "Req Latency (sec)": (5, 7),
             "TTFT (ms)": (8, 10),
             "ITL (ms)": (11, 13),
             "TPOT (ms)": (14, 16),
@@ -944,3 +978,20 @@ class GenerativeBenchmarksConsole:
             title="Benchmarks Stats",
             sections=sections,
         )
+
+    def print_full_report(self):
+        """
+        Print out the benchmark statistics to the console.
+        Temporarily enables the console if it's disabled.
+
+        Format:
+        - Metadata
+        - Info
+        - Stats
+        """
+        orig_enabled = self.enabled
+        self.enabled = True
+        self.print_benchmarks_metadata()
+        self.print_benchmarks_info()
+        self.print_benchmarks_stats()
+        self.enabled = orig_enabled

guidellm/benchmark/profile.py

@@ -17,13 +17,13 @@ from guidellm.scheduler import (
 )
 
 __all__ = [
-    "ProfileType",
+    "AsyncProfile",
+    "ConcurrentProfile",
     "Profile",
+    "ProfileType",
+    "SweepProfile",
     "SynchronousProfile",
-    "ConcurrentProfile",
     "ThroughputProfile",
-    "AsyncProfile",
-    "SweepProfile",
     "create_profile",
 ]
 

guidellm/benchmark/progress.py

@@ -33,10 +33,10 @@ from guidellm.scheduler import (
 from guidellm.utils import Colors
 
 __all__ = [
-    "BenchmarkerTaskProgressState",
     "BenchmarkerProgressDisplay",
-    "GenerativeTextBenchmarkerTaskProgressState",
+    "BenchmarkerTaskProgressState",
     "GenerativeTextBenchmarkerProgressDisplay",
+    "GenerativeTextBenchmarkerTaskProgressState",
 ]
 
 

guidellm/benchmark/scenario.py

@@ -0,0 +1,104 @@
+from collections.abc import Iterable
+from functools import cache
+from pathlib import Path
+from typing import Annotated, Any, Literal, Optional, TypeVar, Union
+
+from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
+from pydantic import BeforeValidator, Field, NonNegativeInt, PositiveFloat, PositiveInt
+from transformers.tokenization_utils_base import (  # type: ignore[import]
+    PreTrainedTokenizerBase,
+)
+
+from guidellm.backend.backend import BackendType
+from guidellm.benchmark.profile import ProfileType
+from guidellm.objects.pydantic import StandardBaseModel
+from guidellm.scheduler.strategy import StrategyType
+
+__ALL__ = ["Scenario", "GenerativeTextScenario", "get_builtin_scenarios"]
+
+SCENARIO_DIR = Path(__file__).parent / "scenarios/"
+
+
+@cache
+def get_builtin_scenarios() -> list[str]:
+    """Returns list of builtin scenario names."""
+    return [p.stem for p in SCENARIO_DIR.glob("*.json")]
+
+
+def parse_float_list(value: Union[str, float, list[float]]) -> list[float]:
+    """
+    Parse a comma separated string to a list of float
+    or convert single float list of one or pass float
+    list through.
+    """
+    if isinstance(value, (int, float)):
+        return [value]
+    elif isinstance(value, list):
+        return value
+
+    values = value.split(",") if "," in value else [value]
+
+    try:
+        return [float(val) for val in values]
+    except ValueError as err:
+        raise ValueError(
+            "must be a number or comma-separated list of numbers."
+        ) from err
+
+
+T = TypeVar("T", bound="Scenario")
+
+
+class Scenario(StandardBaseModel):
+    """
+    Parent Scenario class with common options for all benchmarking types.
+    """
+
+    target: str
+
+    @classmethod
+    def from_builtin(cls: type[T], name: str, overrides: Optional[dict] = None) -> T:
+        filename = SCENARIO_DIR / f"{name}.json"
+
+        if not filename.is_file():
+            raise ValueError(f"{name} is not a valid builtin scenario")
+
+        return cls.from_file(filename, overrides)
+
+
+class GenerativeTextScenario(Scenario):
+    """
+    Scenario class for generative text benchmarks.
+    """
+
+    class Config:
+        # NOTE: This prevents errors due to unvalidatable
+        # types like PreTrainedTokenizerBase
+        arbitrary_types_allowed = True
+
+    backend_type: BackendType = "openai_http"
+    backend_args: Optional[dict[str, Any]] = None
+    model: Optional[str] = None
+    processor: Optional[Union[str, Path, PreTrainedTokenizerBase]] = None
+    processor_args: Optional[dict[str, Any]] = None
+    data: Union[
+        str,
+        Path,
+        Iterable[Union[str, dict[str, Any]]],
+        Dataset,
+        DatasetDict,
+        IterableDataset,
+        IterableDatasetDict,
+    ]
+    data_args: Optional[dict[str, Any]] = None
+    data_sampler: Optional[Literal["random"]] = None
+    rate_type: Union[StrategyType, ProfileType]
+    rate: Annotated[
+        Optional[list[PositiveFloat]], BeforeValidator(parse_float_list)
+    ] = None
+    max_seconds: Optional[PositiveFloat] = None
+    max_requests: Optional[PositiveInt] = None
+    warmup_percent: Annotated[Optional[float], Field(gt=0, le=1)] = None
+    cooldown_percent: Annotated[Optional[float], Field(gt=0, le=1)] = None
+    output_sampling: Optional[NonNegativeInt] = None
+    random_seed: int = 42

guidellm/config.py

@@ -1,4 +1,5 @@
 import json
+import os
 from collections.abc import Sequence
 from enum import Enum
 from typing import Literal, Optional
@@ -11,8 +12,8 @@ __all__ = [
     "Environment",
     "LoggingSettings",
     "OpenAISettings",
-    "print_config",
     "Settings",
+    "print_config",
     "reload_settings",
     "settings",
 ]
@@ -30,10 +31,10 @@ class Environment(str, Enum):
 
 
 ENV_REPORT_MAPPING = {
-    Environment.PROD: "https://guidellm.neuralmagic.com/local-report/index.html",
-    Environment.STAGING: "https://staging.guidellm.neuralmagic.com/local-report/index.html",
-    Environment.DEV: "https://dev.guidellm.neuralmagic.com/local-report/index.html",
-    Environment.LOCAL: "tests/dummy/report.html",
+    Environment.PROD: "https://blog.vllm.ai/guidellm/ui/v0.3.0/index.html",
+    Environment.STAGING: "https://blog.vllm.ai/guidellm/ui/release/v0.3.0/index.html",
+    Environment.DEV: "https://blog.vllm.ai/guidellm/ui/dev/index.html",
+    Environment.LOCAL: "http://localhost:3000/index.html",
 }
 
 
@@ -81,10 +82,24 @@ class OpenAISettings(BaseModel):
 
     api_key: Optional[str] = None
     bearer_token: Optional[str] = None
+    headers: Optional[dict[str, str]] = None
     organization: Optional[str] = None
     project: Optional[str] = None
     base_url: str = "http://localhost:8000"
     max_output_tokens: int = 16384
+    verify: bool = True
+    max_output_key: dict[Literal["text_completions", "chat_completions"], str] = {
+        "text_completions": "max_tokens",
+        "chat_completions": "max_completion_tokens",
+    }
+
+
+class ReportGenerationSettings(BaseModel):
+    """
+    Report generation settings for the application
+    """
+
+    source: str = ""
 
 
 class Settings(BaseSettings):
@@ -115,13 +130,18 @@ class Settings(BaseSettings):
     default_sweep_number: int = 10
 
     # HTTP settings
+    request_follow_redirects: bool = True
     request_timeout: int = 60 * 5  # 5 minutes
     request_http2: bool = True
 
     # Scheduler settings
     max_concurrency: int = 512
-    max_worker_processes: int = 10
-    max_add_requests_per_loop: int = 20
+    max_worker_processes: int = Field(
+        # use number of CPUs - 1, but at least 10
+        default_factory=lambda: max((os.cpu_count() or 1) - 1, 10)
+    )
+    min_queued_requests: int = 20
+    scheduler_start_delay: float = 5
 
     # Data settings
     dataset: DatasetSettings = DatasetSettings()
@@ -139,6 +159,9 @@ class Settings(BaseSettings):
     )
     openai: OpenAISettings = OpenAISettings()
 
+    # Report settings
+    report_generation: ReportGenerationSettings = ReportGenerationSettings()
+
     # Output settings
     table_border_char: str = "="
     table_headers_border_char: str = "-"
@@ -147,6 +170,8 @@ class Settings(BaseSettings):
     @model_validator(mode="after")
     @classmethod
     def set_default_source(cls, values):
+        if not values.report_generation.source:
+            values.report_generation.source = ENV_REPORT_MAPPING.get(values.env)
         return values
 
     def generate_env_file(self) -> str:

guidellm/dataset/__init__.py

@@ -10,13 +10,13 @@ from .synthetic import (
 )
 
 __all__ = [
-    "DatasetCreator",
     "ColumnInputTypes",
-    "HFDatasetsCreator",
-    "load_dataset",
+    "DatasetCreator",
     "FileDatasetCreator",
+    "HFDatasetsCreator",
     "InMemoryDatasetCreator",
-    "SyntheticDatasetCreator",
     "SyntheticDatasetConfig",
+    "SyntheticDatasetCreator",
     "SyntheticTextItemsGenerator",
+    "load_dataset",
 ]

guidellm/dataset/creator.py

@@ -5,7 +5,7 @@ from typing import Any, Literal, Optional, Union
 from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
 from transformers import PreTrainedTokenizerBase  # type: ignore[import]
 
-__all__ = ["DatasetCreator", "ColumnInputTypes"]
+__all__ = ["ColumnInputTypes", "DatasetCreator"]
 
 ColumnInputTypes = Literal[
     "prompt_column",

guidellm/dataset/synthetic.py

@@ -1,6 +1,7 @@
 import json
 import random
 from collections.abc import Iterable, Iterator
+from itertools import cycle
 from pathlib import Path
 from typing import Any, Literal, Optional, Union
 
@@ -18,13 +19,18 @@ from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator
 from guidellm.utils import EndlessTextCreator, IntegerRangeSampler, check_load_processor
 
 __all__ = [
-    "SyntheticDatasetCreator",
     "SyntheticDatasetConfig",
+    "SyntheticDatasetCreator",
     "SyntheticTextItemsGenerator",
 ]
 
 
 class SyntheticDatasetConfig(BaseModel):
+    prefix_tokens: int = Field(
+        description="The number of shared prefix tokens to prepend to each prompt.",
+        ge=0,
+        default=0,
+    )
     prompt_tokens: int = Field(
         description="The average number of text tokens generated for prompts.",
         gt=0,
@@ -163,6 +169,10 @@ class SyntheticTextItemsGenerator(
         )
         # ensure diff distribution from output tokens
         rand = random.Random(self.random_seed + 2)  # noqa: S311
+        unique_prefix_iter = cycle(self.processor.get_vocab().values())
+
+        prefix_index = rand.randint(0, len(self.text_creator.words))
+        prefix_tokens = self._create_prompt(self.config.prefix_tokens, prefix_index)
 
         for _, prompt_tokens, output_tokens in zip(
             range(self.config.samples),
@@ -170,37 +180,52 @@ class SyntheticTextItemsGenerator(
             output_tokens_sampler,
         ):
             start_index = rand.randint(0, len(self.text_creator.words))
+            prompt_text = self.processor.decode(
+                prefix_tokens
+                + self._create_prompt(
+                    prompt_tokens, start_index, next(unique_prefix_iter)
+                ),
+                skip_special_tokens=True,
+            )
             yield {
-                "prompt": self._create_prompt(prompt_tokens, start_index),
-                "prompt_tokens_count": prompt_tokens,
+                "prompt": prompt_text,
+                "prompt_tokens_count": self.config.prefix_tokens + prompt_tokens,
                 "output_tokens_count": output_tokens,
             }
 
-    def _create_prompt(self, prompt_tokens: int, start_index: int) -> str:
+    def _create_prompt(
+        self, prompt_tokens: int, start_index: int, unique_prefix: Optional[int] = None
+    ) -> list[int]:
         if prompt_tokens <= 0:
-            return ""
+            return []
 
         left = start_index
         right = start_index + 4 * prompt_tokens
+        start_tokens = [unique_prefix] if unique_prefix else []
 
         while left < right:
             mid = (left + right) // 2
             test_prompt = self.text_creator.create_text(start_index, mid - start_index)
-            test_tokens = len(self.processor.tokenize(test_prompt))
+            test_tokens = start_tokens + self.processor.encode(test_prompt)
 
-            if test_tokens == prompt_tokens:
-                return test_prompt
-            elif test_tokens < prompt_tokens:
+            if len(test_tokens) == prompt_tokens:
+                return test_tokens
+            elif len(test_tokens) < prompt_tokens:
                 left = mid + 1
             else:
                 right = mid
 
-        return self.text_creator.create_text(start_index, left - start_index)
+        final_text = self.text_creator.create_text(start_index, left - start_index)
+        return start_tokens + self.processor.encode(final_text)
 
 
 class SyntheticDatasetCreator(DatasetCreator):
     @classmethod
-    def is_supported(cls, data: Any, data_args: Optional[dict[str, Any]]) -> bool:  # noqa: ARG003
+    def is_supported(
+        cls,
+        data: Any,
+        data_args: Optional[dict[str, Any]],  # noqa: ARG003
+    ) -> bool:
         if (
             isinstance(data, Path)
             and data.exists()

guidellm/logger.py

@@ -9,13 +9,16 @@ Environment Variables:
     - GUIDELLM__LOGGING__DISABLED: Disable logging (default: false).
     - GUIDELLM__LOGGING__CLEAR_LOGGERS: Clear existing loggers
         from loguru (default: true).
-    - GUIDELLM__LOGGING__LOG_LEVEL: Log level for console logging
+    - GUIDELLM__LOGGING__CONSOLE_LOG_LEVEL: Log level for console logging
         (default: none, options: DEBUG, INFO, WARNING, ERROR, CRITICAL).
-    - GUIDELLM__LOGGING__FILE: Path to the log file for file logging
+    - GUIDELLM__LOGGING__LOG_FILE: Path to the log file for file logging
         (default: guidellm.log if log file level set else none)
-    - GUIDELLM__LOGGING__FILE_LEVEL: Log level for file logging
+    - GUIDELLM__LOGGING__LOG_FILE_LEVEL: Log level for file logging
         (default: INFO if log file set else none).
 
+If logging isn't responding to the environment variables, run the `guidellm config`
+command to validate that the environment variables match and are being set correctly.
+
 Usage:
     from guidellm import logger, configure_logger, LoggerConfig
 
@@ -68,7 +71,8 @@ def configure_logger(config: LoggingSettings = settings.logging):
     logger.add(
         sys.stdout,
         level=config.console_log_level.upper(),
-        format="{time} | {function} | {level} - {message}",
+        format="<green>{time:YY-MM-DD HH:mm:ss}</green>|<level>{level: <8}</level> \
+        |<cyan>{name}:{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
     )
 
     if config.log_file or config.log_file_level:

guidellm/objects/__init__.py

@@ -8,11 +8,11 @@ from .statistics import (
 )
 
 __all__ = [
-    "StandardBaseModel",
-    "StatusBreakdown",
     "DistributionSummary",
     "Percentiles",
     "RunningStats",
+    "StandardBaseModel",
+    "StatusBreakdown",
     "StatusDistributionSummary",
     "TimeRunningStats",
 ]

guidellm/objects/pydantic.py

@@ -1,10 +1,15 @@
-from typing import Any, Generic, TypeVar
+import json
+from pathlib import Path
+from typing import Any, Generic, Optional, TypeVar
 
+import yaml
 from loguru import logger
 from pydantic import BaseModel, ConfigDict, Field
 
 __all__ = ["StandardBaseModel", "StatusBreakdown"]
 
+T = TypeVar("T", bound="StandardBaseModel")
+
 
 class StandardBaseModel(BaseModel):
     """
@@ -27,6 +32,30 @@ class StandardBaseModel(BaseModel):
             data,
         )
 
+    @classmethod
+    def get_default(cls: type[T], field: str) -> Any:
+        """Get default values for model fields"""
+        return cls.model_fields[field].default
+
+    @classmethod
+    def from_file(cls: type[T], filename: Path, overrides: Optional[dict] = None) -> T:
+        """
+        Attempt to create a new instance of the model using
+        data loaded from json or yaml file.
+        """
+        try:
+            with filename.open() as f:
+                if str(filename).endswith(".json"):
+                    data = json.load(f)
+                else:  # Assume everything else is yaml
+                    data = yaml.safe_load(f)
+        except (json.JSONDecodeError, yaml.YAMLError) as e:
+            logger.error(f"Failed to parse {filename} as type {cls.__name__}")
+            raise ValueError(f"Error when parsing file: {filename}") from e
+
+        data.update(overrides)
+        return cls.model_validate(data)
+
 
 SuccessfulT = TypeVar("SuccessfulT")
 ErroredT = TypeVar("ErroredT")

guidellm/objects/statistics.py

@@ -9,10 +9,10 @@ from pydantic import Field, computed_field
 from guidellm.objects.pydantic import StandardBaseModel, StatusBreakdown
 
 __all__ = [
-    "Percentiles",
     "DistributionSummary",
-    "StatusDistributionSummary",
+    "Percentiles",
     "RunningStats",
+    "StatusDistributionSummary",
     "TimeRunningStats",
 ]
 
@@ -37,6 +37,9 @@ class Percentiles(StandardBaseModel):
     p25: float = Field(
         description="The 25th percentile of the distribution.",
     )
+    p50: float = Field(
+        description="The 50th percentile of the distribution.",
+    )
     p75: float = Field(
         description="The 75th percentile of the distribution.",
     )
@@ -159,6 +162,7 @@ class DistributionSummary(StandardBaseModel):
                     p05=cdf[np.argmax(cdf[:, 1] >= 0.05), 0].item(),  # noqa: PLR2004
                     p10=cdf[np.argmax(cdf[:, 1] >= 0.1), 0].item(),  # noqa: PLR2004
                     p25=cdf[np.argmax(cdf[:, 1] >= 0.25), 0].item(),  # noqa: PLR2004
+                    p50=cdf[np.argmax(cdf[:, 1] >= 0.50), 0].item(),  # noqa: PLR2004
                     p75=cdf[np.argmax(cdf[:, 1] >= 0.75), 0].item(),  # noqa: PLR2004
                     p90=cdf[np.argmax(cdf[:, 1] >= 0.9), 0].item(),  # noqa: PLR2004
                     p95=cdf[np.argmax(cdf[:, 1] >= 0.95), 0].item(),  # noqa: PLR2004
@@ -172,6 +176,7 @@ class DistributionSummary(StandardBaseModel):
                     p05=0,
                     p10=0,
                     p25=0,
+                    p50=0,
                     p75=0,
                     p90=0,
                     p95=0,
@@ -238,18 +243,9 @@ class DistributionSummary(StandardBaseModel):
         """
         if distribution_type == "concurrency":
             # convert to delta changes based on when requests were running
-            time_deltas: dict[float, int] = defaultdict(int)
-            for start, end in requests:
-                time_deltas[start] += 1
-                time_deltas[end] -= 1
-
-            # convert to the events over time measuring concurrency changes
-            events = []
-            active = 0
-
-            for time, delta in sorted(time_deltas.items()):
-                active += delta
-                events.append((time, active))
+            events = [(start, 1) for start, _ in requests] + [
+                (end, -1) for _, end in requests
+            ]
         elif distribution_type == "rate":
             # convert to events for when requests finished
             global_start = min(start for start, _ in requests) if requests else 0
@@ -276,6 +272,16 @@ class DistributionSummary(StandardBaseModel):
             else:
                 flattened_events.append((time, val))
 
+        if distribution_type == "concurrency":
+            # convert to the events over time measuring concurrency changes
+            events_over_time: list[tuple[float, float]] = []
+            active = 0
+            for time, delta in flattened_events:
+                active += delta  # type: ignore [assignment]
+                events_over_time.append((time, active))
+
+            flattened_events = events_over_time
+
         # convert to value distribution function
         distribution: dict[float, float] = defaultdict(float)
 

guidellm/preprocess/__init__.py

@@ -0,0 +1,3 @@
+from .dataset import ShortPromptStrategy, process_dataset
+
+__all__ = ["ShortPromptStrategy", "process_dataset"]