guidellm 0.4.0a18__py3-none-any.whl → 0.4.0a155__py3-none-any.whl

guidellm/schemas/stats.py

@@ -0,0 +1,228 @@
+"""
+Request statistics and metrics for generative AI benchmark analysis.
+
+Provides data structures for capturing and analyzing performance metrics from
+generative AI workloads. Contains request-level statistics including token counts,
+latency measurements, and throughput calculations for text generation benchmarks.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import Field, computed_field
+
+from guidellm.schemas.info import RequestInfo
+from guidellm.schemas.request import GenerativeRequestType, UsageMetrics
+from guidellm.utils import StandardBaseDict
+
+__all__ = ["GenerativeRequestStats"]
+
+
+class GenerativeRequestStats(StandardBaseDict):
+    """
+    Request statistics for generative AI text generation workloads.
+
+    Captures comprehensive performance metrics for individual generative requests,
+    including token counts, timing measurements, and derived performance statistics.
+    Provides computed properties for latency analysis, throughput calculations,
+    and token generation metrics essential for benchmark evaluation.
+
+    Example:
+    ::
+        stats = GenerativeRequestStats(
+            request_id="req_123",
+            request_type="text_completion",
+            info=request_info,
+            input_metrics=input_usage,
+            output_metrics=output_usage
+        )
+        throughput = stats.output_tokens_per_second
+    """
+
+    type_: Literal["generative_request_stats"] = "generative_request_stats"
+    request_id: str = Field(description="Unique identifier for the request")
+    request_type: GenerativeRequestType | str = Field(
+        description="Type of generative request: text or chat completion"
+    )
+    request_args: str | None = Field(
+        default=None, description="Arguments passed to the backend for this request"
+    )
+    output: str | None = Field(
+        description="Generated text output, if request completed successfully"
+    )
+    info: RequestInfo = Field(
+        description="Metadata and timing information for the request"
+    )
+    input_metrics: UsageMetrics = Field(
+        description="Usage statistics for the input prompt"
+    )
+    output_metrics: UsageMetrics = Field(
+        description="Usage statistics for the generated output"
+    )
+
+    # Request stats
+    @computed_field  # type: ignore[misc]
+    @property
+    def request_latency(self) -> float | None:
+        """
+        End-to-end request processing latency in seconds.
+
+        :return: Duration from request start to completion, or None if unavailable.
+        """
+        if not self.info.timings.request_end or not self.info.timings.request_start:
+            return None
+
+        return self.info.timings.request_end - self.info.timings.request_start
+
+    # General token stats
+    @computed_field  # type: ignore[misc]
+    @property
+    def prompt_tokens(self) -> int | None:
+        """
+        Number of tokens in the input prompt.
+
+        :return: Input prompt token count, or None if unavailable.
+        """
+        return self.input_metrics.text_tokens
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def input_tokens(self) -> int | None:
+        """
+        Number of tokens in the input prompt.
+
+        :return: Input prompt token count, or None if unavailable.
+        """
+        return self.input_metrics.total_tokens
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def output_tokens(self) -> int | None:
+        """
+        Number of tokens in the generated output.
+
+        :return: Generated output token count, or None if unavailable.
+        """
+        return self.output_metrics.total_tokens
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_tokens(self) -> int | None:
+        """
+        Total token count including prompt and output tokens.
+
+        :return: Sum of prompt and output tokens, or None if either is unavailable.
+        """
+        input_tokens = self.input_metrics.total_tokens
+        output_tokens = self.output_metrics.total_tokens
+
+        if input_tokens is None and output_tokens is None:
+            return None
+
+        return (input_tokens or 0) + (output_tokens or 0)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def time_to_first_token_ms(self) -> float | None:
+        """
+        Time to first token generation in milliseconds.
+
+        :return: Latency from request start to first token, or None if unavailable.
+        """
+        if (
+            not self.info.timings.first_iteration
+            or not self.info.timings.request_start
+            or self.info.timings.first_iteration == self.info.timings.last_iteration
+        ):
+            return None
+
+        return 1000 * (
+            self.info.timings.first_iteration - self.info.timings.request_start
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def time_per_output_token_ms(self) -> float | None:
+        """
+        Average time per output token in milliseconds.
+
+        Includes time for first token and all subsequent tokens.
+
+        :return: Average milliseconds per output token, or None if unavailable.
+        """
+        if (
+            not self.info.timings.request_start
+            or not self.info.timings.last_iteration
+            or not self.output_metrics.total_tokens
+        ):
+            return None
+
+        return (
+            1000
+            * (self.info.timings.last_iteration - self.info.timings.request_start)
+            / self.output_metrics.total_tokens
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def inter_token_latency_ms(self) -> float | None:
+        """
+        Average inter-token latency in milliseconds.
+
+        Measures time between token generations, excluding first token.
+
+        :return: Average milliseconds between tokens, or None if unavailable.
+        """
+        if (
+            not self.info.timings.first_iteration
+            or not self.info.timings.last_iteration
+            or not self.output_metrics.total_tokens
+            or self.output_metrics.total_tokens <= 1
+        ):
+            return None
+
+        return (
+            1000
+            * (self.info.timings.last_iteration - self.info.timings.first_iteration)
+            / (self.output_metrics.total_tokens - 1)
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def tokens_per_second(self) -> float | None:
+        """
+        Overall token throughput including prompt and output tokens.
+
+        :return: Total tokens per second, or None if unavailable.
+        """
+        if not (latency := self.request_latency) or self.total_tokens is None:
+            return None
+
+        return self.total_tokens / latency
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def output_tokens_per_second(self) -> float | None:
+        """
+        Output token generation throughput.
+
+        :return: Output tokens per second, or None if unavailable.
+        """
+        if not (latency := self.request_latency) or self.output_tokens is None:
+            return None
+
+        return self.output_tokens / latency
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def output_tokens_per_iteration(self) -> float | None:
+        """
+        Average output tokens generated per iteration.
+
+        :return: Output tokens per iteration, or None if unavailable.
+        """
+        if self.output_tokens is None or not self.info.timings.iterations:
+            return None
+
+        return self.output_tokens / self.info.timings.iterations

guidellm/{config.py → settings.py}

@@ -1,8 +1,9 @@
+from __future__ import annotations
+
 import json
-import os
 from collections.abc import Sequence
 from enum import Enum
-from typing import Literal, Optional
+from typing import Literal
 
 from pydantic import BaseModel, Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -46,8 +47,8 @@ class LoggingSettings(BaseModel):
     disabled: bool = False
     clear_loggers: bool = True
     console_log_level: str = "WARNING"
-    log_file: Optional[str] = None
-    log_file_level: Optional[str] = None
+    log_file: str | None = None
+    log_file_level: str | None = None
 
 
 class DatasetSettings(BaseModel):
@@ -80,14 +81,18 @@ class OpenAISettings(BaseModel):
     for OpenAI server based pathways
     """
 
-    api_key: Optional[str] = None
-    bearer_token: Optional[str] = None
-    headers: Optional[dict[str, str]] = None
-    organization: Optional[str] = None
-    project: Optional[str] = None
+    api_key: str | None = None
+    bearer_token: str | None = None
+    headers: dict[str, str] | None = None
+    organization: str | None = None
+    project: str | None = None
     base_url: str = "http://localhost:8000"
     max_output_tokens: int = 16384
     verify: bool = True
+    max_output_key: dict[Literal["text_completions", "chat_completions"], str] = {
+        "text_completions": "max_tokens",
+        "chat_completions": "max_completion_tokens",
+    }
 
 
 class ReportGenerationSettings(BaseModel):
@@ -131,24 +136,30 @@ class Settings(BaseSettings):
     request_http2: bool = True
 
     # Scheduler settings
+    mp_context_type: Literal["spawn", "fork", "forkserver"] | None = "fork"
+    mp_serialization: Literal["dict", "sequence"] | None = "dict"
+    mp_encoding: (
+        Literal["msgpack", "msgspec"]
+        | None
+        | list[Literal["msgpack", "msgspec"] | None]
+    ) = ["msgspec", "msgpack", None]
+    mp_messaging_object: Literal["queue", "manager_queue", "pipe"] = "queue"
+    mp_requests_send_buffer_size: int = 1
+    mp_poll_interval: float = 0.1
+    mp_max_pending_buffer_percent: float = 0.5
+    mp_max_worker_buffer_percent: float = 0.2
     max_concurrency: int = 512
-    max_worker_processes: int = Field(
-        # use number of CPUs - 1, but at least 10
-        default_factory=lambda: max((os.cpu_count() or 1) - 1, 10)
-    )
-    min_queued_requests: int = 20
-    scheduler_start_delay: float = 5
+    max_worker_processes: int = 10
+    scheduler_start_delay_non_distributed: float = 1.0
+    constraint_error_window_size: float = 30
+    constraint_error_min_processed: float = 30
 
     # Data settings
     dataset: DatasetSettings = DatasetSettings()
 
     # Request/stats settings
-    preferred_prompt_tokens_source: Optional[
-        Literal["request", "response", "local"]
-    ] = "response"
-    preferred_output_tokens_source: Optional[
-        Literal["request", "response", "local"]
-    ] = "response"
+    preferred_prompt_tokens_source: Literal["request", "response"] = "response"
+    preferred_output_tokens_source: Literal["request", "response"] = "response"
     preferred_backend: Literal["openai"] = "openai"
     preferred_route: Literal["text_completions", "chat_completions"] = (
         "text_completions"

guidellm/utils/__init__.py

@@ -1,39 +1,126 @@
-from .colors import Colors
+from .auto_importer import AutoImporterMixin
+from .console import Colors, Console, ConsoleUpdateStep, StatusIcons, StatusStyles
 from .default_group import DefaultGroupHandler
 from .dict import recursive_key_update
-from .hf_datasets import (
-    SUPPORTED_TYPES,
-    save_dataset_to_file,
+from .encoding import (
+    Encoder,
+    EncodingTypesAlias,
+    MessageEncoding,
+    SerializationTypesAlias,
+    Serializer,
 )
-from .hf_transformers import (
-    check_load_processor,
+from .functions import (
+    all_defined,
+    safe_add,
+    safe_divide,
+    safe_format_timestamp,
+    safe_getattr,
+    safe_multiply,
+)
+from .hf_datasets import SUPPORTED_TYPES, save_dataset_to_file
+from .hf_transformers import check_load_processor
+from .imports import json
+from .messaging import (
+    InterProcessMessaging,
+    InterProcessMessagingManagerQueue,
+    InterProcessMessagingPipe,
+    InterProcessMessagingQueue,
+    SendMessageT,
+)
+from .mixins import InfoMixin
+from .pydantic_utils import (
+    PydanticClassRegistryMixin,
+    ReloadableBaseModel,
+    StandardBaseDict,
+    StandardBaseModel,
+    StatusBreakdown,
 )
 from .random import IntegerRangeSampler
+from .registry import RegistryMixin, RegistryObjT
+from .singleton import SingletonMixin, ThreadSafeSingletonMixin
+from .statistics import (
+    DistributionSummary,
+    Percentiles,
+    RunningStats,
+    StatusDistributionSummary,
+    TimeRunningStats,
+)
+from .synchronous import (
+    wait_for_sync_barrier,
+    wait_for_sync_event,
+    wait_for_sync_objects,
+)
 from .text import (
     EndlessTextCreator,
     camelize_str,
     clean_text,
     filter_text,
-    is_puncutation,
+    format_value_display,
+    is_punctuation,
     load_text,
     split_text,
     split_text_list_by_length,
 )
+from .typing import get_literal_vals
 
 __all__ = [
     "SUPPORTED_TYPES",
+    "AutoImporterMixin",
+    "Colors",
     "Colors",
+    "Console",
+    "ConsoleUpdateStep",
     "DefaultGroupHandler",
+    "DistributionSummary",
+    "Encoder",
+    "EncodingTypesAlias",
     "EndlessTextCreator",
+    "InfoMixin",
     "IntegerRangeSampler",
+    "InterProcessMessaging",
+    "InterProcessMessagingManagerQueue",
+    "InterProcessMessagingPipe",
+    "InterProcessMessagingQueue",
+    "MessageEncoding",
+    "MessageEncoding",
+    "Percentiles",
+    "PydanticClassRegistryMixin",
+    "RegistryMixin",
+    "RegistryObjT",
+    "ReloadableBaseModel",
+    "RunningStats",
+    "SendMessageT",
+    "SerializationTypesAlias",
+    "Serializer",
+    "SingletonMixin",
+    "StandardBaseDict",
+    "StandardBaseModel",
+    "StatusBreakdown",
+    "StatusDistributionSummary",
+    "StatusIcons",
+    "StatusStyles",
+    "ThreadSafeSingletonMixin",
+    "TimeRunningStats",
+    "all_defined",
     "camelize_str",
     "check_load_processor",
     "clean_text",
     "filter_text",
-    "is_puncutation",
+    "format_value_display",
+    "get_literal_vals",
+    "is_punctuation",
+    "json",
     "load_text",
     "recursive_key_update",
+    "safe_add",
+    "safe_divide",
+    "safe_format_timestamp",
+    "safe_getattr",
+    "safe_multiply",
     "save_dataset_to_file",
     "split_text",
     "split_text_list_by_length",
+    "wait_for_sync_barrier",
+    "wait_for_sync_event",
+    "wait_for_sync_objects",
 ]

guidellm/utils/auto_importer.py

@@ -0,0 +1,98 @@
+"""
+Automatic module importing utilities for dynamic class discovery.
+
+This module provides a mixin class for automatic module importing within a package,
+enabling dynamic discovery of classes and implementations without explicit imports.
+It is particularly useful for auto-registering classes in a registry pattern where
+subclasses need to be discoverable at runtime.
+
+The AutoImporterMixin can be combined with registration mechanisms to create
+extensible systems where new implementations are automatically discovered and
+registered when they are placed in the correct package structure.
+"""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+import sys
+from typing import ClassVar
+
+__all__ = ["AutoImporterMixin"]
+
+
+class AutoImporterMixin:
+    """
+    Mixin class for automatic module importing within packages.
+
+    This mixin enables dynamic discovery of classes and implementations without
+    explicit imports by automatically importing all modules within specified
+    packages. It is designed for use with class registration mechanisms to enable
+    automatic discovery and registration of classes when they are placed in the
+    correct package structure.
+
+    Example:
+    ::
+        from guidellm.utils import AutoImporterMixin
+
+        class MyRegistry(AutoImporterMixin):
+            auto_package = "my_package.implementations"
+
+        MyRegistry.auto_import_package_modules()
+
+    :cvar auto_package: Package name or tuple of package names to import modules from
+    :cvar auto_ignore_modules: Module names to ignore during import
+    :cvar auto_imported_modules: List tracking which modules have been imported
+    """
+
+    auto_package: ClassVar[str | tuple[str, ...] | None] = None
+    auto_ignore_modules: ClassVar[tuple[str, ...] | None] = None
+    auto_imported_modules: ClassVar[list[str] | None] = None
+
+    @classmethod
+    def auto_import_package_modules(cls) -> None:
+        """
+        Automatically import all modules within the specified package(s).
+
+        Scans the package(s) defined in the `auto_package` class variable and imports
+        all modules found, tracking them in `auto_imported_modules`. Skips packages
+        (directories) and any modules listed in `auto_ignore_modules`.
+
+        :raises ValueError: If the `auto_package` class variable is not set
+        """
+        if cls.auto_package is None:
+            raise ValueError(
+                "The class variable 'auto_package' must be set to the package name to "
+                "import modules from."
+            )
+
+        cls.auto_imported_modules = []
+        packages = (
+            cls.auto_package
+            if isinstance(cls.auto_package, tuple)
+            else (cls.auto_package,)
+        )
+
+        for package_name in packages:
+            package = importlib.import_module(package_name)
+
+            for _, module_name, is_pkg in pkgutil.walk_packages(
+                package.__path__, package.__name__ + "."
+            ):
+                if (
+                    is_pkg
+                    or (
+                        cls.auto_ignore_modules is not None
+                        and module_name in cls.auto_ignore_modules
+                    )
+                    or module_name in cls.auto_imported_modules
+                ):
+                    # Skip packages and ignored modules
+                    continue
+
+                if module_name in sys.modules:
+                    # Avoid circular imports
+                    cls.auto_imported_modules.append(module_name)
+                else:
+                    importlib.import_module(module_name)
+                    cls.auto_imported_modules.append(module_name)

guidellm/utils/cli.py

@@ -3,10 +3,31 @@ from typing import Any
 
 import click
 
+__all__ = ["Union", "format_list_arg", "parse_json", "set_if_not_default"]
+
 
 def parse_json(ctx, param, value):  # noqa: ARG001
-    if value is None:
+    if value is None or value == [None]:
         return None
+    if isinstance(value, list | tuple):
+        return [parse_json(ctx, param, val) for val in value]
+
+    if "{" not in value and "}" not in value and "=" in value:
+        # Treat it as a key=value pair if it doesn't look like JSON.
+        result = {}
+        for pair in value.split(","):
+            if "=" not in pair:
+                raise click.BadParameter(
+                    f"{param.name} must be a valid JSON string or key=value pairs."
+                )
+            key, val = pair.split("=", 1)
+            result[key.strip()] = val.strip()
+        return result
+
+    if "{" not in value and "}" not in value:
+        # Treat it as a plain string if it doesn't look like JSON.
+        return value
+
     try:
         return json.loads(value)
     except json.JSONDecodeError as err:
@@ -26,6 +47,29 @@ def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]:
     return values
 
 
+def format_list_arg(
+    value: Any, default: Any = None, simplify_single: bool = False
+) -> list[Any] | Any:
+    """
+    Format a multi-argument value for display.
+
+    :param value: The value to format, which can be a single value or a list/tuple.
+    :param default: The default value to set if the value is non truthy.
+    :param simplify_single: If True and the value is a single-item list/tuple,
+        return the single item instead of a list.
+    :return: Formatted list of values, or single value if simplify_single and applicable
+    """
+    if not value:
+        return default
+
+    if isinstance(value, tuple):
+        value = list(value)
+    elif not isinstance(value, list):
+        value = [value]
+
+    return value if not simplify_single or len(value) != 1 else value[0]
+
+
 class Union(click.ParamType):
     """
     A custom click parameter type that allows for multiple types to be accepted.
@@ -35,7 +79,7 @@ class Union(click.ParamType):
         self.types = types
         self.name = "".join(t.name for t in types)
 
-    def convert(self, value, param, ctx):  # noqa: RET503
+    def convert(self, value, param, ctx):
         fails = []
         for t in self.types:
             try: