guidellm 0.3.1__py3-none-any.whl → 0.6.0a5__py3-none-any.whl

guidellm/{config.py → settings.py}

@@ -1,8 +1,9 @@
+from __future__ import annotations
+
 import json
-import os
 from collections.abc import Sequence
 from enum import Enum
-from typing import Literal, Optional
+from typing import Literal
 
 from pydantic import BaseModel, Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -31,8 +32,8 @@ class Environment(str, Enum):
 
 
 ENV_REPORT_MAPPING = {
-    Environment.PROD: "https://blog.vllm.ai/guidellm/ui/v0.3.0/index.html",
-    Environment.STAGING: "https://blog.vllm.ai/guidellm/ui/release/v0.3.0/index.html",
+    Environment.PROD: "https://blog.vllm.ai/guidellm/ui/v0.5.0/index.html",
+    Environment.STAGING: "https://blog.vllm.ai/guidellm/ui/release/v0.4.0/index.html",
     Environment.DEV: "https://blog.vllm.ai/guidellm/ui/dev/index.html",
     Environment.LOCAL: "http://localhost:3000/index.html",
 }
@@ -46,8 +47,8 @@ class LoggingSettings(BaseModel):
     disabled: bool = False
     clear_loggers: bool = True
     console_log_level: str = "WARNING"
-    log_file: Optional[str] = None
-    log_file_level: Optional[str] = None
+    log_file: str | None = None
+    log_file_level: str | None = None
 
 
 class DatasetSettings(BaseModel):
@@ -80,11 +81,11 @@ class OpenAISettings(BaseModel):
     for OpenAI server based pathways
     """
 
-    api_key: Optional[str] = None
-    bearer_token: Optional[str] = None
-    headers: Optional[dict[str, str]] = None
-    organization: Optional[str] = None
-    project: Optional[str] = None
+    api_key: str | None = None
+    bearer_token: str | None = None
+    headers: dict[str, str] | None = None
+    organization: str | None = None
+    project: str | None = None
     base_url: str = "http://localhost:8000"
     max_output_tokens: int = 16384
     verify: bool = True
@@ -135,27 +136,33 @@ class Settings(BaseSettings):
     request_http2: bool = True
 
     # Scheduler settings
+    mp_context_type: Literal["spawn", "fork", "forkserver"] | None = "fork"
+    mp_serialization: Literal["dict", "sequence"] | None = "dict"
+    mp_encoding: (
+        Literal["msgpack", "msgspec"]
+        | None
+        | list[Literal["msgpack", "msgspec"] | None]
+    ) = ["msgspec", "msgpack", None]
+    mp_messaging_object: Literal["queue", "manager_queue", "pipe"] = "queue"
+    mp_requests_send_buffer_size: int = 1
+    mp_poll_interval: float = 0.1
+    mp_max_pending_buffer_percent: float = 0.5
+    mp_max_worker_buffer_percent: float = 0.2
     max_concurrency: int = 512
-    max_worker_processes: int = Field(
-        # use number of CPUs - 1, but at least 10
-        default_factory=lambda: max((os.cpu_count() or 1) - 1, 10)
-    )
-    min_queued_requests: int = 20
-    scheduler_start_delay: float = 5
+    max_worker_processes: int = 10
+    scheduler_start_delay_non_distributed: float = 1.0
+    constraint_error_window_size: float = 30
+    constraint_error_min_processed: float = 30
 
     # Data settings
     dataset: DatasetSettings = DatasetSettings()
 
     # Request/stats settings
-    preferred_prompt_tokens_source: Optional[
-        Literal["request", "response", "local"]
-    ] = "response"
-    preferred_output_tokens_source: Optional[
-        Literal["request", "response", "local"]
-    ] = "response"
+    preferred_prompt_tokens_source: Literal["request", "response"] = "response"
+    preferred_output_tokens_source: Literal["request", "response"] = "response"
     preferred_backend: Literal["openai"] = "openai"
     preferred_route: Literal["text_completions", "chat_completions"] = (
-        "text_completions"
+        "chat_completions"
     )
     openai: OpenAISettings = OpenAISettings()
 

guidellm/utils/__init__.py

@@ -1,39 +1,102 @@
-from .colors import Colors
+from .auto_importer import AutoImporterMixin
+from .console import Colors, Console, ConsoleUpdateStep, StatusIcons, StatusStyles
 from .default_group import DefaultGroupHandler
 from .dict import recursive_key_update
-from .hf_datasets import (
-    SUPPORTED_TYPES,
-    save_dataset_to_file,
+from .encoding import (
+    Encoder,
+    EncodingTypesAlias,
+    MessageEncoding,
+    SerializationTypesAlias,
+    Serializer,
 )
-from .hf_transformers import (
-    check_load_processor,
+from .functions import (
+    all_defined,
+    safe_add,
+    safe_divide,
+    safe_format_number,
+    safe_format_timestamp,
+    safe_getattr,
+    safe_multiply,
 )
+from .hf_datasets import SUPPORTED_TYPES, save_dataset_to_file
+from .hf_transformers import check_load_processor
+from .imports import json
+from .messaging import (
+    InterProcessMessaging,
+    InterProcessMessagingManagerQueue,
+    InterProcessMessagingPipe,
+    InterProcessMessagingQueue,
+    SendMessageT,
+)
+from .mixins import InfoMixin
 from .random import IntegerRangeSampler
+from .registry import RegistryMixin, RegistryObjT
+from .singleton import SingletonMixin, ThreadSafeSingletonMixin
+from .synchronous import (
+    wait_for_sync_barrier,
+    wait_for_sync_event,
+    wait_for_sync_objects,
+)
 from .text import (
     EndlessTextCreator,
     camelize_str,
     clean_text,
     filter_text,
-    is_puncutation,
+    format_value_display,
+    is_punctuation,
     load_text,
     split_text,
     split_text_list_by_length,
 )
+from .typing import get_literal_vals
 
 __all__ = [
     "SUPPORTED_TYPES",
+    "AutoImporterMixin",
     "Colors",
+    "Console",
+    "ConsoleUpdateStep",
     "DefaultGroupHandler",
+    "Encoder",
+    "EncodingTypesAlias",
     "EndlessTextCreator",
+    "InfoMixin",
     "IntegerRangeSampler",
+    "InterProcessMessaging",
+    "InterProcessMessagingManagerQueue",
+    "InterProcessMessagingPipe",
+    "InterProcessMessagingQueue",
+    "MessageEncoding",
+    "RegistryMixin",
+    "RegistryObjT",
+    "SendMessageT",
+    "SerializationTypesAlias",
+    "Serializer",
+    "SingletonMixin",
+    "StatusIcons",
+    "StatusStyles",
+    "ThreadSafeSingletonMixin",
+    "all_defined",
     "camelize_str",
     "check_load_processor",
     "clean_text",
     "filter_text",
-    "is_puncutation",
+    "format_value_display",
+    "get_literal_vals",
+    "is_punctuation",
+    "json",
     "load_text",
     "recursive_key_update",
+    "safe_add",
+    "safe_divide",
+    "safe_format_number",
+    "safe_format_timestamp",
+    "safe_getattr",
+    "safe_multiply",
     "save_dataset_to_file",
     "split_text",
     "split_text_list_by_length",
+    "wait_for_sync_barrier",
+    "wait_for_sync_event",
+    "wait_for_sync_objects",
 ]

guidellm/utils/auto_importer.py

@@ -0,0 +1,98 @@
+"""
+Automatic module importing utilities for dynamic class discovery.
+
+This module provides a mixin class for automatic module importing within a package,
+enabling dynamic discovery of classes and implementations without explicit imports.
+It is particularly useful for auto-registering classes in a registry pattern where
+subclasses need to be discoverable at runtime.
+
+The AutoImporterMixin can be combined with registration mechanisms to create
+extensible systems where new implementations are automatically discovered and
+registered when they are placed in the correct package structure.
+"""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+import sys
+from typing import ClassVar
+
+__all__ = ["AutoImporterMixin"]
+
+
+class AutoImporterMixin:
+    """
+    Mixin class for automatic module importing within packages.
+
+    This mixin enables dynamic discovery of classes and implementations without
+    explicit imports by automatically importing all modules within specified
+    packages. It is designed for use with class registration mechanisms to enable
+    automatic discovery and registration of classes when they are placed in the
+    correct package structure.
+
+    Example:
+    ::
+        from guidellm.utils import AutoImporterMixin
+
+        class MyRegistry(AutoImporterMixin):
+            auto_package = "my_package.implementations"
+
+        MyRegistry.auto_import_package_modules()
+
+    :cvar auto_package: Package name or tuple of package names to import modules from
+    :cvar auto_ignore_modules: Module names to ignore during import
+    :cvar auto_imported_modules: List tracking which modules have been imported
+    """
+
+    auto_package: ClassVar[str | tuple[str, ...] | None] = None
+    auto_ignore_modules: ClassVar[tuple[str, ...] | None] = None
+    auto_imported_modules: ClassVar[list[str] | None] = None
+
+    @classmethod
+    def auto_import_package_modules(cls) -> None:
+        """
+        Automatically import all modules within the specified package(s).
+
+        Scans the package(s) defined in the `auto_package` class variable and imports
+        all modules found, tracking them in `auto_imported_modules`. Skips packages
+        (directories) and any modules listed in `auto_ignore_modules`.
+
+        :raises ValueError: If the `auto_package` class variable is not set
+        """
+        if cls.auto_package is None:
+            raise ValueError(
+                "The class variable 'auto_package' must be set to the package name to "
+                "import modules from."
+            )
+
+        cls.auto_imported_modules = []
+        packages = (
+            cls.auto_package
+            if isinstance(cls.auto_package, tuple)
+            else (cls.auto_package,)
+        )
+
+        for package_name in packages:
+            package = importlib.import_module(package_name)
+
+            for _, module_name, is_pkg in pkgutil.walk_packages(
+                package.__path__, package.__name__ + "."
+            ):
+                if (
+                    is_pkg
+                    or (
+                        cls.auto_ignore_modules is not None
+                        and module_name in cls.auto_ignore_modules
+                    )
+                    or module_name in cls.auto_imported_modules
+                ):
+                    # Skip packages and ignored modules
+                    continue
+
+                if module_name in sys.modules:
+                    # Avoid circular imports
+                    cls.auto_imported_modules.append(module_name)
+                else:
+                    importlib.import_module(module_name)
+                    cls.auto_imported_modules.append(module_name)

guidellm/utils/cli.py

@@ -1,12 +1,116 @@
+import contextlib
 import json
+import os
 from typing import Any
 
 import click
 
+__all__ = [
+    "Union",
+    "format_list_arg",
+    "list_set_env",
+    "parse_json",
+    "parse_list",
+    "parse_list_floats",
+    "set_if_not_default",
+]
 
-def parse_json(ctx, param, value):  # noqa: ARG001
-    if value is None:
+
+def list_set_env(prefix: str = "GUIDELLM_") -> list[str]:
+    """
+    List all set environment variables prefixed with the given prefix.
+
+    :param prefix: The prefix to filter environment variables.
+    :return: List of environment variable names that are set with the given prefix.
+    """
+    return [key for key in os.environ if key.startswith(prefix)]
+
+
+def parse_list(ctx, param, value) -> list[str] | None:
+    """
+    Click callback to parse the input value into a list of strings.
+    Supports single strings, comma-separated strings,
+    and lists/tuples of any of these formats (when multiple=True is used).
+
+    :param ctx: Click context
+    :param param: Click parameter
+    :param value: The input value to parse
+    :return: List of parsed strings
+    """
+    if value is None or value == [None]:
+        # Handle null values directly or nested null (when multiple=True)
         return None
+
+    if isinstance(value, list | tuple):
+        # Handle multiple=True case by recursively parsing each item and combining
+        parsed = []
+        for val in value:
+            if (items := parse_list(ctx, param, val)) is not None:
+                parsed.extend(items)
+        return parsed
+
+    if isinstance(value, str) and "," in value:
+        # Handle comma-separated strings
+        return [item.strip() for item in value.split(",") if item.strip()]
+
+    if isinstance(value, str):
+        # Handle single string
+        return [value.strip()]
+
+    # Fall back to returning as a single-item list
+    return [value]
+
+
+def parse_list_floats(ctx, param, value):
+    str_list = parse_list(ctx, param, value)
+    if str_list is None:
+        return None
+
+    item = None  # For error reporting
+    try:
+        return [float(item) for item in str_list]
+    except ValueError as err:
+        # Raise a Click error if any part isn't a valid float
+        raise click.BadParameter(
+            f"Input '{value}' is not a valid comma-separated list "
+            f"of floats/ints. Failed on {item} Error: {err}"
+        ) from err
+
+
+def parse_json(ctx, param, value):  # noqa: ARG001, C901, PLR0911
+    if isinstance(value, dict):
+        return value
+
+    if value is None or value == [None]:
+        return None
+
+    if isinstance(value, str) and not value.strip():
+        return None
+
+    if isinstance(value, list | tuple):
+        return [parse_json(ctx, param, val) for val in value]
+
+    if "{" not in value and "}" not in value and "=" in value:
+        # Treat it as a key=value pair if it doesn't look like JSON.
+        result = {}
+        for pair in value.split(","):
+            if "=" not in pair:
+                raise click.BadParameter(
+                    f"{param.name} must be a valid JSON string or key=value pairs."
+                )
+            key, val = pair.split("=", 1)
+            result[key.strip()] = val.strip()
+        return result
+
+    if "{" not in value and "}" not in value:
+        # Treat it as a primitive if it doesn't look like JSON.
+        try:
+            value = int(value)
+        except ValueError:
+            with contextlib.suppress(ValueError):
+                value = float(value)
+        return value
+
     try:
         return json.loads(value)
     except json.JSONDecodeError as err:
@@ -26,6 +130,29 @@ def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]:
     return values
 
 
+def format_list_arg(
+    value: Any, default: Any = None, simplify_single: bool = False
+) -> list[Any] | Any:
+    """
+    Format a multi-argument value for display.
+
+    :param value: The value to format, which can be a single value or a list/tuple.
+    :param default: The default value to set if the value is non truthy.
+    :param simplify_single: If True and the value is a single-item list/tuple,
+        return the single item instead of a list.
+    :return: Formatted list of values, or single value if simplify_single and applicable
+    """
+    if not value:
+        return default
+
+    if isinstance(value, tuple):
+        value = list(value)
+    elif not isinstance(value, list):
+        value = [value]
+
+    return value if not simplify_single or len(value) != 1 else value[0]
+
+
 class Union(click.ParamType):
     """
     A custom click parameter type that allows for multiple types to be accepted.
@@ -35,7 +162,7 @@ class Union(click.ParamType):
         self.types = types
         self.name = "".join(t.name for t in types)
 
-    def convert(self, value, param, ctx):  # noqa: RET503
+    def convert(self, value, param, ctx):
         fails = []
         for t in self.types:
             try:
@@ -46,9 +173,9 @@ class Union(click.ParamType):
 
         self.fail("; ".join(fails) or f"Invalid value: {value}")  # noqa: RET503
 
-    def get_metavar(self, param: click.Parameter) -> str:
+    def get_metavar(self, param: click.Parameter, ctx: click.Context) -> str:
         def get_choices(t: click.ParamType) -> str:
-            meta = t.get_metavar(param)
+            meta = t.get_metavar(param, ctx)
             return meta if meta is not None else t.name
 
         # Get the choices for each type in the union.