guidellm 0.3.1__py3-none-any.whl → 0.6.0a5__py3-none-any.whl

guidellm/schemas/request.py

@@ -0,0 +1,235 @@
+"""
+Request schema definitions for generation operations.
+
+Contains request models and data structures used to define and execute generation
+requests across different backend services. Provides standardized interfaces for
+request arguments, usage metrics tracking, and request type definitions that enable
+consistent interaction with various AI generation APIs.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Literal
+
+from pydantic import Field, computed_field
+
+from guidellm.schemas.base import StandardBaseDict, StandardBaseModel
+
+__all__ = [
+    "GenerationRequest",
+    "GenerationRequestArguments",
+    "GenerativeRequestType",
+    "UsageMetrics",
+]
+
+
+GenerativeRequestType = Literal[
+    "text_completions",
+    "chat_completions",
+    "audio_transcriptions",
+    "audio_translations",
+]
+
+
+class GenerationRequestArguments(StandardBaseDict):
+    """
+    HTTP request arguments for generation operations.
+
+    Encapsulates all necessary HTTP request components including method, headers,
+    parameters, and payload data required to execute generation requests against
+    backend services. Supports file uploads and streaming responses.
+    """
+
+    method: str | None = Field(
+        default=None,
+        description="The HTTP method to use for the request (e.g., 'POST', 'GET').",
+    )
+    stream: bool | None = Field(
+        default=None,
+        description="Whether to stream the response, if applicable.",
+    )
+    headers: dict[str, str] | None = Field(
+        default=None,
+        description="Any headers to include in the request, if applicable.",
+    )
+    params: dict[str, Any] | None = Field(
+        default=None,
+        description="Query parameters to include in the request, if applicable.",
+    )
+    body: dict[str, Any] | None = Field(
+        default=None,
+        description="Content to include in the main request body.",
+    )
+    files: dict[str, Any] | None = Field(
+        default=None,
+        description="Files to include in the request, if applicable.",
+    )
+
+    def model_combine(
+        self, additional: GenerationRequestArguments | dict[str, Any]
+    ) -> GenerationRequestArguments:
+        """
+        Merge additional request arguments into the current instance.
+
+        Combines method and stream fields by overwriting, while merging collection
+        fields like headers, params, body, and files by extending existing values.
+
+        :param additional: Additional arguments to merge with current instance
+        :return: Updated instance with merged arguments
+        """
+        additional_dict = (
+            additional.model_dump()
+            if isinstance(additional, GenerationRequestArguments)
+            else additional
+        )
+
+        for overwrite in ("method", "stream"):
+            if (val := additional_dict.get(overwrite)) is not None:
+                setattr(self, overwrite, val)
+
+        for combine in ("headers", "params", "body", "files"):
+            if (val := additional_dict.get(combine)) is not None:
+                current = getattr(self, combine, None) or {}
+                setattr(self, combine, {**current, **val})
+
+        return self
+
+
+class UsageMetrics(StandardBaseDict):
+    """
+    Multimodal usage metrics for generation requests.
+
+    Tracks resource consumption across different modalities including text, images,
+    video, and audio. Provides granular metrics for tokens, bytes, duration, and
+    format-specific measurements to enable comprehensive usage monitoring and billing.
+    """
+
+    # Text stats
+    text_tokens: int | None = Field(
+        default=None, description="Number of text tokens processed/generated."
+    )
+    text_words: int | None = Field(
+        default=None, description="Number of text words processed/generated."
+    )
+    text_characters: int | None = Field(
+        default=None, description="Number of text characters processed/generated."
+    )
+
+    # Vision image stats
+    image_tokens: int | None = Field(
+        default=None, description="Number of image tokens processed/generated."
+    )
+    image_count: int | None = Field(
+        default=None, description="Number of images processed/generated."
+    )
+    image_pixels: int | None = Field(
+        default=None, description="Number of image pixels processed/generated."
+    )
+    image_bytes: int | None = Field(
+        default=None, description="Number of image bytes processed/generated."
+    )
+
+    # Vision video stats
+    video_tokens: int | None = Field(
+        default=None, description="Number of video tokens processed/generated."
+    )
+    video_frames: int | None = Field(
+        default=None, description="Number of video frames processed/generated."
+    )
+    video_seconds: float | None = Field(
+        default=None, description="Duration of video processed/generated in seconds."
+    )
+    video_bytes: int | None = Field(
+        default=None, description="Number of video bytes processed/generated."
+    )
+
+    # Audio stats
+    audio_tokens: int | None = Field(
+        default=None, description="Number of audio tokens processed/generated."
+    )
+    audio_samples: int | None = Field(
+        default=None, description="Number of audio samples processed/generated."
+    )
+    audio_seconds: float | None = Field(
+        default=None, description="Duration of audio processed/generated in seconds."
+    )
+    audio_bytes: int | None = Field(
+        default=None, description="Number of audio bytes processed/generated."
+    )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_tokens(self) -> int | None:
+        """
+        Calculate total tokens across all modalities.
+
+        :return: Sum of text, image, video, and audio tokens, or None if all are None
+        """
+        token_metrics = [
+            self.text_tokens,
+            self.image_tokens,
+            self.video_tokens,
+            self.audio_tokens,
+        ]
+        # NOTE: None should indicate no data rather than zero usage
+        if token_metrics.count(None) == len(token_metrics):
+            return None
+        else:
+            return sum(token or 0 for token in token_metrics)
+
+    def add_text_metrics(self, text):
+        """
+        Adds the metrics from the given text to the fields
+        `text_characters` and `text_words`.
+
+        :param text: Text to add metrics from
+        """
+        self.text_characters = (self.text_characters or 0) + len(text)
+        self.text_words = (self.text_words or 0) + len(text.split())
+
+
+class GenerationRequest(StandardBaseModel):
+    """
+    Complete request specification for backend generation operations.
+
+    Encapsulates all components needed to execute a generation request including
+    unique identification, request type specification, HTTP arguments, and input/output
+    usage metrics. Serves as the primary interface between the scheduler and backend
+    services for coordinating AI generation tasks.
+
+    Example::
+        request = GenerationRequest(
+            request_type="text_completions",
+            arguments=GenerationRequestArguments(
+                method="POST",
+                body={"prompt": "Hello world", "max_tokens": 100}
+            )
+        )
+    """
+
+    request_id: str = Field(
+        default_factory=lambda: str(uuid.uuid4()),
+        description="Unique identifier for the request.",
+    )
+    request_type: GenerativeRequestType | str = Field(
+        description=(
+            "Type of request. If url is not provided in arguments, "
+            "this will be used to determine the request url."
+        ),
+    )
+    arguments: GenerationRequestArguments = Field(
+        description=(
+            "Payload for the request, structured as a dictionary of arguments to pass "
+            "to the respective backend method. For example, can contain "
+            "'json', 'headers', 'files', etc."
+        )
+    )
+    input_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Input statistics including counts, sizes, and durations.",
+    )
+    output_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Output statistics including counts, sizes, and durations.",
+    )

guidellm/schemas/request_stats.py

@@ -0,0 +1,349 @@
+"""
+Request statistics and metrics for generative AI benchmark analysis.
+
+Provides data structures for capturing and analyzing performance metrics from
+generative AI workloads. The module contains request-level statistics including
+token counts, latency measurements, and throughput calculations essential for
+evaluating text generation benchmark performance. Computed properties enable
+analysis of time-to-first-token, inter-token latency, and token generation rates.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+import numpy as np
+from pydantic import Field, computed_field
+
+from guidellm.schemas.base import StandardBaseDict
+from guidellm.schemas.info import RequestInfo
+from guidellm.schemas.request import GenerativeRequestType, UsageMetrics
+
+__all__ = ["GenerativeRequestStats"]
+
+
+class GenerativeRequestStats(StandardBaseDict):
+    """
+    Request statistics for generative AI text generation workloads.
+
+    Captures comprehensive performance metrics for individual generative requests,
+    including token counts, timing measurements, and derived performance statistics.
+    Provides computed properties for latency analysis, throughput calculations,
+    and token generation metrics essential for benchmark evaluation.
+
+    Example:
+    ::
+        stats = GenerativeRequestStats(
+            request_id="req_123",
+            request_type="text_completion",
+            info=request_info,
+            input_metrics=input_usage,
+            output_metrics=output_usage
+        )
+        throughput = stats.output_tokens_per_second
+    """
+
+    type_: Literal["generative_request_stats"] = "generative_request_stats"
+    request_id: str = Field(description="Unique identifier for the request")
+    request_type: GenerativeRequestType | str = Field(
+        description="Type of generative request (text_completion or chat_completion)"
+    )
+    response_id: str | None = Field(
+        default=None, description="Unique identifier matching vLLM Response ID"
+    )
+    request_args: str | None = Field(
+        default=None, description="Backend arguments used for this request"
+    )
+    output: str | None = Field(
+        default=None, description="Generated text output from the request"
+    )
+    info: RequestInfo = Field(description="Request metadata and timing information")
+    input_metrics: UsageMetrics = Field(
+        description="Token usage statistics for the input prompt"
+    )
+    output_metrics: UsageMetrics = Field(
+        description="Token usage statistics for the generated output"
+    )
+
+    # Request stats
+    @computed_field  # type: ignore[misc]
+    @property
+    def request_start_time(self) -> float | None:
+        """
+        :return: Timestamp when the request started, or None if unavailable
+        """
+        return (
+            self.info.timings.request_start
+            if self.info.timings.request_start is not None
+            else self.info.timings.resolve_start
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def request_end_time(self) -> float:
+        """
+        :return: Timestamp when the request ended, or None if unavailable
+        """
+        if self.info.timings.resolve_end is None:
+            raise ValueError("resolve_end timings should be set but is None.")
+
+        return (
+            self.info.timings.request_end
+            if self.info.timings.request_end is not None
+            else self.info.timings.resolve_end
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def request_latency(self) -> float | None:
+        """
+        End-to-end request processing latency in seconds.
+
+        :return: Duration from request start to completion, or None if unavailable
+        """
+        start = self.info.timings.request_start
+        end = self.info.timings.request_end
+        if start is None or end is None:
+            return None
+
+        return end - start
+
+    # General token stats
+    @computed_field  # type: ignore[misc]
+    @property
+    def prompt_tokens(self) -> int | None:
+        """
+        :return: Number of tokens in the input prompt, or None if unavailable
+        """
+        return self.input_metrics.total_tokens
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def output_tokens(self) -> int | None:
+        """
+        :return: Number of tokens in the generated output, or None if unavailable
+        """
+        # Fallback if we did not get usage metrics from the server
+        # NOTE: This assumes each iteration is one token
+        if self.output_metrics.total_tokens is None:
+            return self.info.timings.token_iterations or None
+
+        return self.output_metrics.total_tokens
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_tokens(self) -> int | None:
+        """
+        :return: Sum of prompt and output tokens, or None if both unavailable
+        """
+        input_tokens = self.prompt_tokens
+        output_tokens = self.output_tokens
+
+        if input_tokens is None and output_tokens is None:
+            return None
+
+        return (input_tokens or 0) + (output_tokens or 0)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def time_to_first_token_ms(self) -> float | None:
+        """
+        :return: Time to first token generation in milliseconds, or None if unavailable
+        """
+        first_token = self.first_token_iteration
+        start = self.info.timings.request_start
+        if first_token is None or start is None:
+            return None
+
+        return 1000 * (first_token - start)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def time_per_output_token_ms(self) -> float | None:
+        """
+        Average time per output token in milliseconds including first token.
+
+        :return: Average milliseconds per output token, or None if unavailable
+        """
+        if (
+            (start := self.info.timings.request_start) is None
+            or (
+                (last_token := self.last_token_iteration or self.request_end_time)
+                is None
+            )
+            or (output_tokens := self.output_tokens) is None
+            or output_tokens == 0
+        ):
+            return None
+
+        return 1000 * (last_token - start) / output_tokens
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def inter_token_latency_ms(self) -> float | None:
+        """
+        Average inter-token latency in milliseconds excluding first token.
+
+        :return: Average milliseconds between token generations, or None if unavailable
+        """
+        first_token = self.first_token_iteration
+        last_token = self.last_token_iteration
+        output_tokens = self.output_tokens
+        if (
+            first_token is None
+            or last_token is None
+            or output_tokens is None
+            or output_tokens <= 1
+        ):
+            return None
+
+        return 1000 * (last_token - first_token) / (output_tokens - 1)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def tokens_per_second(self) -> float | None:
+        """
+        :return: Total tokens per second throughput, or None if unavailable
+        """
+        if not (latency := self.request_latency) or self.total_tokens is None:
+            return None
+
+        return self.total_tokens / latency
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def output_tokens_per_second(self) -> float | None:
+        """
+        :return: Output token generation throughput, or None if unavailable
+        """
+        if not (latency := self.request_latency) or self.output_tokens is None:
+            return None
+
+        return self.output_tokens / latency
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def iter_tokens_per_iteration(self) -> float | None:
+        """
+        :return: Average tokens per iteration excluding first token, or None if
+            unavailable
+        """
+        if (
+            self.output_tokens is None
+            or self.output_tokens <= 1
+            or self.token_iterations <= 1
+        ):
+            return None
+
+        return (self.output_tokens - 1.0) / (
+            self.token_iterations - 1.0
+        )  # subtract 1 for first token from the prompt, assume first iter is 1 token
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def output_tokens_per_iteration(self) -> float | None:
+        """
+        :return: Average output tokens per iteration, or None if unavailable
+        """
+        if self.output_tokens is None or self.token_iterations < 1:
+            return None
+
+        return self.output_tokens / self.token_iterations
+
+    @property
+    def first_token_iteration(self) -> float | None:
+        """
+        :return: Timestamp of first token generation, or None if unavailable
+        """
+        return self.info.timings.first_token_iteration
+
+    @property
+    def last_token_iteration(self) -> float | None:
+        """
+        :return: Timestamp of last token generation, or None if unavailable
+        """
+        return self.info.timings.last_token_iteration
+
+    @property
+    def token_iterations(self) -> int:
+        """
+        :return: Total number of token generation iterations
+        """
+        return self.info.timings.token_iterations
+
+    @property
+    def prompt_tokens_timing(self) -> tuple[float, float]:
+        """
+        :return: Tuple of (timestamp, token_count) for prompt processing
+        :raises ValueError: If resolve_end timings are not set
+        """
+        return (
+            (
+                self.first_token_iteration
+                if self.first_token_iteration is not None
+                else self.request_end_time
+            ),
+            self.prompt_tokens or 0.0,
+        )
+
+    @property
+    def output_tokens_timings(self) -> list[tuple[float, float]]:
+        """
+        :return: List of (timestamp, token_count) tuples for output token generations
+        :raises ValueError: If resolve_end timings are not set
+        """
+        if (
+            self.first_token_iteration is None
+            or self.last_token_iteration is None
+            or self.token_iterations <= 1
+        ):
+            # No iteration data, return single timing at end with all tokens
+            return [
+                (
+                    (
+                        self.last_token_iteration
+                        if self.last_token_iteration is not None
+                        else self.request_end_time
+                    ),
+                    self.output_tokens or 0.0,
+                )
+            ]
+
+        # Return first token timing as 1 token plus per-iteration timings
+        return [
+            (self.first_token_iteration, 1.0 * bool(self.output_tokens))
+        ] + self.iter_tokens_timings
+
+    @property
+    def iter_tokens_timings(self) -> list[tuple[float, float]]:
+        """
+        :return: List of (timestamp, token_count) tuples for iterations excluding
+            first token
+        """
+        if (
+            self.first_token_iteration is None
+            or self.last_token_iteration is None
+            or (tok_per_iter := self.iter_tokens_per_iteration) is None
+            or self.token_iterations <= 1
+        ):
+            return []
+
+        # evenly space the iterations since we don't have per-iteration timings
+        # / we don't know the individual token counts per iteration
+        iter_times = np.linspace(
+            self.first_token_iteration,
+            self.last_token_iteration,
+            num=self.token_iterations,
+        )[1:]  # skip first iteration
+
+        return [(iter_time, tok_per_iter) for iter_time in iter_times]
+
+    @property
+    def total_tokens_timings(self) -> list[tuple[float, float]]:
+        """
+        :return: List of (timestamp, token_count) tuples for all token generations
+        """
+        prompt_timings = self.prompt_tokens_timing
+        output_timings = self.output_tokens_timings
+
+        return ([prompt_timings] if prompt_timings else []) + output_timings

guidellm/schemas/response.py

@@ -0,0 +1,124 @@
+"""
+Backend response models for request and response handling.
+
+Provides standardized response models for generation operations that capture
+output text, usage metrics, and compilation of request statistics. Ensures
+consistent data handling and statistics aggregation across different backend
+implementations.
+"""
+
+from __future__ import annotations
+
+from pydantic import Field
+
+from guidellm.schemas.base import StandardBaseModel
+from guidellm.schemas.info import RequestInfo
+from guidellm.schemas.request import GenerationRequest, UsageMetrics
+from guidellm.schemas.request_stats import GenerativeRequestStats
+
+__all__ = ["GenerationResponse"]
+
+
+class GenerationResponse(StandardBaseModel):
+    """
+    Response model for backend generation operations.
+
+    Captures the output and metrics from a generation request, providing structured
+    data for text output, token usage statistics, and compilation of detailed
+    request statistics for analysis and monitoring purposes.
+
+    Example:
+    ::
+        response = GenerationResponse(
+            request_id="req-123",
+            text="Generated response text",
+            input_metrics=UsageMetrics(token_count=50),
+            output_metrics=UsageMetrics(token_count=25)
+        )
+        stats = response.compile_stats(request, info)
+    """
+
+    request_id: str = Field(
+        description="Unique identifier matching the original GenerationRequest."
+    )
+    response_id: str | None = Field(
+        default=None,
+        description="Unique identifier matching the original vLLM Response ID.",
+    )
+    request_args: str | None = Field(
+        description="Arguments passed to the backend for request processing."
+    )
+    text: str | None = Field(
+        default=None,
+        description="The generated response text.",
+    )
+    input_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Token usage statistics from the input prompt.",
+    )
+    output_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Token usage statistics from the generated output.",
+    )
+
+    def compile_stats(
+        self,
+        request: GenerationRequest,
+        info: RequestInfo,
+        prefer_response: bool = True,
+    ) -> GenerativeRequestStats:
+        """
+        Compile and return comprehensive request statistics.
+
+        Merges metrics from the request and response objects to create a complete
+        statistical record, with preference given to response-level metrics when
+        available to ensure accuracy of actual execution data.
+
+        :param request: The original generation request containing input data
+        :param info: Metadata and timing information for the request execution
+        :param prefer_response: Whether to prefer response metrics over request
+            metrics when both are available
+        :return: A GenerativeRequestStats object containing detailed statistics
+        :raises ValueError: When request IDs don't match between objects
+        """
+        if request.request_id != self.request_id:
+            raise ValueError("Mismatched request IDs between request and response.")
+
+        if info.request_id != self.request_id:
+            raise ValueError("Mismatched request IDs between info and response.")
+
+        if info.status != "completed":
+            # clear out request output metrics if the request failed since
+            # those are not valid
+            request.output_metrics = UsageMetrics()
+
+        base_input = request.input_metrics if prefer_response else self.input_metrics
+        override_input = (
+            self.input_metrics if prefer_response else request.input_metrics
+        )
+        base_output = request.output_metrics if prefer_response else self.output_metrics
+        override_output = (
+            self.output_metrics if prefer_response else request.output_metrics
+        )
+
+        input_metrics_dict = base_input.model_dump()
+        for key, value in override_input.model_dump().items():
+            if value is not None:
+                input_metrics_dict[key] = value
+        output_metrics_dict = base_output.model_dump()
+        for key, value in override_output.model_dump().items():
+            if value is not None:
+                output_metrics_dict[key] = value
+
+        return GenerativeRequestStats(
+            request_id=self.request_id,
+            response_id=self.response_id,
+            request_type=request.request_type,
+            request_args=str(
+                request.arguments.model_dump() if request.arguments else {}
+            ),
+            output=self.text,
+            info=info,
+            input_metrics=UsageMetrics(**input_metrics_dict),
+            output_metrics=UsageMetrics(**output_metrics_dict),
+        )