guidellm 0.3.0rc20250507__py3-none-any.whl → 0.4.0a0__py3-none-any.whl

guidellm/scheduler/worker.py

@@ -1,11 +1,11 @@
 import asyncio
 import math
-import multiprocessing
-import multiprocessing.queues
 import time
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator
 from dataclasses import dataclass
+from itertools import islice
+from threading import Event
 from typing import (
     Any,
     Generic,
@@ -26,36 +26,24 @@ from guidellm.backend import (
 )
 from guidellm.objects import StandardBaseModel
 from guidellm.request import GenerationRequest
-from guidellm.scheduler.result import SchedulerRequestInfo
-from guidellm.scheduler.types import RequestT, ResponseT
+from guidellm.request.types import RequestT, ResponseT
+from guidellm.scheduler.queues import MPQueues, Queue, QueueEmpty
+from guidellm.scheduler.result import (
+    SchedulerRequestInfo,
+    WorkerProcessRequest,
+    WorkerProcessResult,
+)
+from guidellm.scheduler.strategy import SchedulingStrategy
 
 __all__ = [
-    "WorkerProcessRequest",
-    "WorkerProcessResult",
+    "GenerativeRequestsWorker",
+    "GenerativeRequestsWorkerDescription",
+    "RequestsWorker",
     "ResolveStatus",
     "WorkerDescription",
-    "RequestsWorker",
-    "GenerativeRequestsWorkerDescription",
-    "GenerativeRequestsWorker",
 ]
 
 
-@dataclass
-class WorkerProcessRequest(Generic[RequestT]):
-    request: RequestT
-    start_time: float
-    timeout_time: float
-    queued_time: float
-
-
-@dataclass
-class WorkerProcessResult(Generic[RequestT, ResponseT]):
-    type_: Literal["request_scheduled", "request_start", "request_complete"]
-    request: RequestT
-    response: Optional[ResponseT]
-    info: SchedulerRequestInfo
-
-
 @dataclass
 class ResolveStatus:
     requested: bool
@@ -120,28 +108,25 @@ class RequestsWorker(ABC, Generic[RequestT, ResponseT]):
         """
         ...
 
-    async def get_request(
-        self, requests_queue: multiprocessing.Queue
-    ) -> Optional[WorkerProcessRequest[RequestT]]:
-        return await asyncio.to_thread(requests_queue.get)  # type: ignore[attr-defined]
-
     async def send_result(
         self,
-        results_queue: multiprocessing.Queue,
+        results_queue: Queue[WorkerProcessResult[RequestT, ResponseT]],
         result: WorkerProcessResult[RequestT, ResponseT],
     ):
         await asyncio.to_thread(results_queue.put, result)  # type: ignore[attr-defined]
 
     async def resolve_scheduler_request(
         self,
-        request: Any,
-        queued_time: float,
+        process_request: WorkerProcessRequest[RequestT, ResponseT],
         dequeued_time: float,
         start_time: float,
-        timeout_time: float,
-        results_queue: multiprocessing.Queue,
+        results_queue: Queue[WorkerProcessResult[RequestT, ResponseT]],
         process_id: int,
     ):
+        request = process_request.request
+        timeout_time = process_request.timeout_time
+        queued_time = process_request.queued_time
+
         info = SchedulerRequestInfo(
             targeted_start_time=start_time,
             queued_time=queued_time,
@@ -185,74 +170,57 @@ class RequestsWorker(ABC, Generic[RequestT, ResponseT]):
         )
         asyncio.create_task(self.send_result(results_queue, result))
 
-    def process_loop_synchronous(
-        self,
-        requests_queue: multiprocessing.Queue,
-        results_queue: multiprocessing.Queue,
-        process_id: int,
-    ):
-        async def _process_runner():
-            while (
-                process_request := await self.get_request(requests_queue)
-            ) is not None:
-                dequeued_time = time.time()
-
-                await self.resolve_scheduler_request(
-                    request=process_request.request,
-                    queued_time=process_request.queued_time,
-                    dequeued_time=dequeued_time,
-                    start_time=process_request.start_time,
-                    timeout_time=process_request.timeout_time,
-                    results_queue=results_queue,
-                    process_id=process_id,
-                )
-
-        try:
-            asyncio.run(_process_runner())
-        except Exception as exc:  # noqa: BLE001
-            logger.error(
-                f"Error in worker process {process_id}: {exc}",
-                exc_info=True,
-                stack_info=True,
-            )
-
     def process_loop_asynchronous(
         self,
-        requests_queue: multiprocessing.Queue,
-        results_queue: multiprocessing.Queue,
+        queues: MPQueues[RequestT, ResponseT],
+        strategy: SchedulingStrategy,
+        stop_event: Event,
         max_concurrency: int,
         process_id: int,
+        num_processes: int,
     ):
         async def _process_runner():
-            pending = asyncio.Semaphore(max_concurrency)
+            lock = asyncio.Semaphore(max_concurrency)
+            times_iter = islice(
+                strategy.request_times(),
+                process_id,
+                None,
+                num_processes,
+            )
 
-            if pending.locked():
-                raise ValueError("Async worker called with max_concurrency < 1")
+            start_time = None
+            while not stop_event.is_set():
+                if start_time is None:
+                    start_time = next(times_iter)
 
-            while (
-                process_request := await self.get_request(requests_queue)
-            ) is not None:
-                dequeued_time = time.time()
+                # Yield control to the event loop. Sleep if we are way ahead
+                await asyncio.sleep(start_time - time.time() - 1)
+                await lock.acquire()
 
-                await pending.acquire()
+                try:
+                    process_request = queues.requests.get_nowait()
+                    dequeued_time = time.time()
+                except QueueEmpty:
+                    lock.release()
+                    continue
 
-                def _task_done(_: asyncio.Task):
-                    nonlocal pending
-                    pending.release()
+                def _request_callback(
+                    _: asyncio.Future[WorkerProcessRequest[RequestT, ResponseT]],
+                ):
+                    nonlocal lock
+                    lock.release()
 
                 task = asyncio.create_task(
                     self.resolve_scheduler_request(
-                        request=process_request.request,
-                        queued_time=process_request.queued_time,
+                        process_request=process_request,
                         dequeued_time=dequeued_time,
-                        start_time=process_request.start_time,
-                        timeout_time=process_request.timeout_time,
-                        results_queue=results_queue,
+                        start_time=start_time,
+                        results_queue=queues.responses,
                         process_id=process_id,
                     )
                 )
-                task.add_done_callback(_task_done)
-                await asyncio.sleep(0)  # enable start task immediately
+                task.add_done_callback(_request_callback)
+                start_time = None
 
         try:
             asyncio.run(_process_runner())
@@ -309,32 +277,23 @@ class GenerativeRequestsWorker(RequestsWorker[GenerationRequest, ResponseSummary
         """
         await self.backend.prepare_multiprocessing()
 
-    def process_loop_synchronous(
-        self,
-        requests_queue: multiprocessing.Queue,
-        results_queue: multiprocessing.Queue,
-        process_id: int,
-    ):
-        asyncio.run(self.backend.validate())
-        super().process_loop_synchronous(
-            requests_queue=requests_queue,
-            results_queue=results_queue,
-            process_id=process_id,
-        )
-
     def process_loop_asynchronous(
         self,
-        requests_queue: multiprocessing.Queue,
-        results_queue: multiprocessing.Queue,
+        queues: MPQueues[GenerationRequest, ResponseSummary],
+        strategy: SchedulingStrategy,
+        stop_event: Event,
         max_concurrency: int,
         process_id: int,
+        num_processes: int,
     ):
         asyncio.run(self.backend.validate())
         super().process_loop_asynchronous(
-            requests_queue=requests_queue,
-            results_queue=results_queue,
+            queues=queues,
+            strategy=strategy,
+            stop_event=stop_event,
             max_concurrency=max_concurrency,
             process_id=process_id,
+            num_processes=num_processes,
         )
 
     async def resolve(
@@ -475,6 +434,7 @@ class GenerativeRequestsWorker(RequestsWorker[GenerationRequest, ResponseSummary
                 request_args=RequestArgs(
                     target=self.backend.target,
                     headers={},
+                    params={},
                     payload={},
                 ),
                 start_time=resolve_start_time,
@@ -490,6 +450,7 @@ class GenerativeRequestsWorker(RequestsWorker[GenerationRequest, ResponseSummary
                 request_args=RequestArgs(
                     target=self.backend.target,
                     headers={},
+                    params={},
                     payload={},
                 ),
                 start_time=response.start_time,

guidellm/utils/__init__.py

@@ -1,10 +1,17 @@
 from .colors import Colors
+from .default_group import DefaultGroupHandler
+from .dict import recursive_key_update
+from .hf_datasets import (
+    SUPPORTED_TYPES,
+    save_dataset_to_file,
+)
 from .hf_transformers import (
     check_load_processor,
 )
 from .random import IntegerRangeSampler
 from .text import (
     EndlessTextCreator,
+    camelize_str,
     clean_text,
     filter_text,
     is_puncutation,
@@ -14,14 +21,19 @@ from .text import (
 )
 
 __all__ = [
-    "IntegerRangeSampler",
+    "SUPPORTED_TYPES",
     "Colors",
+    "DefaultGroupHandler",
+    "EndlessTextCreator",
+    "IntegerRangeSampler",
+    "camelize_str",
     "check_load_processor",
-    "filter_text",
     "clean_text",
-    "split_text",
-    "load_text",
+    "filter_text",
     "is_puncutation",
-    "EndlessTextCreator",
+    "load_text",
+    "recursive_key_update",
+    "save_dataset_to_file",
+    "split_text",
     "split_text_list_by_length",
 ]

guidellm/utils/cli.py

@@ -0,0 +1,62 @@
+import json
+from typing import Any
+
+import click
+
+
+def parse_json(ctx, param, value):  # noqa: ARG001
+    if value is None:
+        return None
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError as err:
+        raise click.BadParameter(f"{param.name} must be a valid JSON string.") from err
+
+
+def set_if_not_default(ctx: click.Context, **kwargs) -> dict[str, Any]:
+    """
+    Set the value of a click option if it is not the default value.
+    This is useful for setting options that are not None by default.
+    """
+    values = {}
+    for k, v in kwargs.items():
+        if ctx.get_parameter_source(k) != click.core.ParameterSource.DEFAULT:  # type: ignore[attr-defined]
+            values[k] = v
+
+    return values
+
+
+class Union(click.ParamType):
+    """
+    A custom click parameter type that allows for multiple types to be accepted.
+    """
+
+    def __init__(self, *types: click.ParamType):
+        self.types = types
+        self.name = "".join(t.name for t in types)
+
+    def convert(self, value, param, ctx):  # noqa: RET503
+        fails = []
+        for t in self.types:
+            try:
+                return t.convert(value, param, ctx)
+            except click.BadParameter as e:
+                fails.append(str(e))
+                continue
+
+        self.fail("; ".join(fails) or f"Invalid value: {value}")  # noqa: RET503
+
+    def get_metavar(self, param: click.Parameter) -> str:
+        def get_choices(t: click.ParamType) -> str:
+            meta = t.get_metavar(param)
+            return meta if meta is not None else t.name
+
+        # Get the choices for each type in the union.
+        choices_str = "|".join(map(get_choices, self.types))
+
+        # Use curly braces to indicate a required argument.
+        if param.required and param.param_type_name == "argument":
+            return f"{{{choices_str}}}"
+
+        # Use square braces to indicate an option or optional argument.
+        return f"[{choices_str}]"

guidellm/utils/default_group.py

@@ -0,0 +1,105 @@
+"""
+File uses code adapted from code with the following license:
+
+Copyright (c) 2015-2023, Heungsub Lee
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+  Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+  Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+__all__ = ["DefaultGroupHandler"]
+
+import collections.abc as cabc
+
+import click
+
+
+class DefaultGroupHandler(click.Group):
+    """
+    Allows the migration to a new sub-command by allowing the group to run
+    one of its sub-commands as the no-args default command.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # To resolve as the default command.
+        if not kwargs.get("ignore_unknown_options", True):
+            raise ValueError("Default group accepts unknown options")
+        self.ignore_unknown_options = True
+        self.default_cmd_name = kwargs.pop("default", None)
+        self.default_if_no_args = kwargs.pop("default_if_no_args", False)
+        super().__init__(*args, **kwargs)
+
+    def parse_args(self, ctx, args):
+        if not args and self.default_if_no_args:
+            args.insert(0, self.default_cmd_name)
+        return super().parse_args(ctx, args)
+
+    def get_command(self, ctx, cmd_name):
+        if cmd_name not in self.commands:
+            # If it doesn't match an existing command, use the default command name.
+            ctx.arg0 = cmd_name
+            cmd_name = self.default_cmd_name
+        return super().get_command(ctx, cmd_name)
+
+    def resolve_command(self, ctx, args):
+        cmd_name, cmd, args = super().resolve_command(ctx, args)
+        if hasattr(ctx, "arg0"):
+            args.insert(0, ctx.arg0)
+            if cmd is not None:
+                cmd_name = cmd.name
+        return cmd_name, cmd, args
+
+    def format_commands(self, ctx, formatter):
+        """
+        Used to wrap the default formatter to clarify which command is the default.
+        """
+        formatter = DefaultCommandFormatter(self, formatter, mark=" (default)")
+        return super().format_commands(ctx, formatter)
+
+
+class DefaultCommandFormatter(click.HelpFormatter):
+    """
+    Wraps a formatter to edit the line for the default command to mark it
+    with the specified mark string.
+    """
+
+    def __init__(self, group, formatter, mark="*"):
+        self.group = group
+        self.formatter = formatter
+        self.mark = mark
+        super().__init__()
+
+    def __getattr__(self, attr):
+        return getattr(self.formatter, attr)
+
+    def write_dl(self, rows: cabc.Sequence[tuple[str, str]], *args, **kwargs):
+        rows_: list[tuple[str, str]] = []
+        for cmd_name, help_msg in rows:
+            if cmd_name == self.group.default_cmd_name:
+                rows_.insert(0, (cmd_name + self.mark, help_msg))
+            else:
+                rows_.append((cmd_name, help_msg))
+        return self.formatter.write_dl(rows_, *args, **kwargs)

guidellm/utils/dict.py

@@ -0,0 +1,23 @@
+def recursive_key_update(d, key_update_func):
+    if not isinstance(d, dict) and not isinstance(d, list):
+        return d
+
+    if isinstance(d, list):
+        for item in d:
+            recursive_key_update(item, key_update_func)
+        return d
+
+    updated_key_pairs = []
+    for key, _ in d.items():
+        updated_key = key_update_func(key)
+        if key != updated_key:
+            updated_key_pairs.append((key, updated_key))
+
+    for key_pair in updated_key_pairs:
+        old_key, updated_key = key_pair
+        d[updated_key] = d[old_key]
+        del d[old_key]
+
+    for _, value in d.items():
+        recursive_key_update(value, key_update_func)
+    return d

guidellm/utils/hf_datasets.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+from typing import Union
+
+from datasets import Dataset
+
+SUPPORTED_TYPES = {
+    ".json",
+    ".jsonl",
+    ".csv",
+    ".parquet",
+}
+
+
+def save_dataset_to_file(dataset: Dataset, output_path: Union[str, Path]) -> None:
+    """
+    Saves a HuggingFace Dataset to file in a supported format.
+
+    :param dataset: Dataset to save.
+    :param output_path: Output file path (.json, .jsonl, .csv, .parquet).
+    :raises ValueError: If the file extension is not supported.
+    """
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    suffix = output_path.suffix.lower()
+
+    if suffix == ".csv":
+        dataset.to_csv(output_path)
+    elif suffix in {".json", ".jsonl"}:
+        dataset.to_json(output_path)
+    elif suffix == ".parquet":
+        dataset.to_parquet(output_path)
+    else:
+        raise ValueError(
+            f"Unsupported file suffix '{suffix}' in output_path'{output_path}'."
+            f" Only {SUPPORTED_TYPES} are supported."
+        )

guidellm/utils/random.py

@@ -37,7 +37,7 @@ class IntegerRangeSampler:
             if calc_min == calc_max:
                 yield calc_min
             elif not self.variance:
-                yield self.rng.randint(calc_min, calc_max + 1)
+                yield self.rng.randint(calc_min, calc_max)
             else:
                 rand = self.rng.gauss(self.average, self.variance)
                 yield round(max(calc_min, min(calc_max, rand)))

guidellm/utils/text.py

@@ -13,13 +13,14 @@ from guidellm import data as package_data
 from guidellm.config import settings
 
 __all__ = [
-    "split_text_list_by_length",
-    "filter_text",
+    "EndlessTextCreator",
+    "camelize_str",
     "clean_text",
-    "split_text",
-    "load_text",
+    "filter_text",
     "is_puncutation",
-    "EndlessTextCreator",
+    "load_text",
+    "split_text",
+    "split_text_list_by_length",
 ]
 
 MAX_PATH_LENGTH = 4096
@@ -189,6 +190,12 @@ def is_puncutation(text: str) -> bool:
     return len(text) == 1 and not text.isalnum() and not text.isspace()
 
 
+def camelize_str(snake_case_string: str) -> str:
+    return (words := snake_case_string.split("_"))[0].lower() + "".join(
+        word.capitalize() for word in words[1:]
+    )
+
+
 class EndlessTextCreator:
     def __init__(
         self,

guidellm/version.py

@@ -0,0 +1,6 @@
+version = "0.1.0a0"
+build_type = "nightly"
+build_iteration = "0"
+git_commit = "None"
+git_branch = "None"
+git_last_tag = "None"