guidellm 0.4.0a18__py3-none-any.whl → 0.4.0a155__py3-none-any.whl

guidellm/schemas/__init__.py

@@ -0,0 +1,31 @@
+"""
+Pydantic schema models for GuideLLM operations.
+
+Provides standardized data models and type definitions for generation requests,
+responses, timing measurements, and statistics aggregation. These schemas ensure
+type safety and consistent data handling across the benchmarking pipeline,
+from request submission through backend processing to results compilation.
+"""
+
+from __future__ import annotations
+
+from .info import RequestInfo, RequestTimings
+from .request import (
+    GenerationRequest,
+    GenerationRequestArguments,
+    GenerativeRequestType,
+    UsageMetrics,
+)
+from .response import GenerationResponse
+from .stats import GenerativeRequestStats
+
+__all__ = [
+    "GenerationRequest",
+    "GenerationRequestArguments",
+    "GenerationResponse",
+    "GenerativeRequestStats",
+    "GenerativeRequestType",
+    "RequestInfo",
+    "RequestTimings",
+    "UsageMetrics",
+]

guidellm/schemas/info.py

@@ -0,0 +1,159 @@
+"""
+Core data structures and interfaces for the GuideLLM scheduler system.
+
+Provides type-safe abstractions for distributed request processing, timing
+measurements, and backend interfaces for benchmarking operations. Central to
+the scheduler architecture, enabling request lifecycle tracking, backend
+coordination, and state management across distributed worker processes.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Literal
+
+from pydantic import Field, computed_field
+
+from guidellm.utils import StandardBaseDict, StandardBaseModel
+
+__all__ = ["RequestInfo", "RequestTimings"]
+
+
+class RequestTimings(StandardBaseDict):
+    """
+    Timing measurements for tracking request lifecycle events.
+
+    Provides comprehensive timing data for distributed request processing, capturing
+    key timestamps from initial targeting through final completion. Essential for
+    performance analysis, SLA monitoring, and debugging request processing bottlenecks
+    across scheduler workers and backend systems.
+    """
+
+    targeted_start: float | None = Field(
+        default=None,
+        description="Unix timestamp when request was initially targeted for execution",
+    )
+    queued: float | None = Field(
+        default=None,
+        description="Unix timestamp when request was placed into processing queue",
+    )
+    dequeued: float | None = Field(
+        default=None,
+        description="Unix timestamp when request was removed from queue for processing",
+    )
+    scheduled_at: float | None = Field(
+        default=None,
+        description="Unix timestamp when the request was scheduled for processing",
+    )
+    resolve_start: float | None = Field(
+        default=None,
+        description="Unix timestamp when backend resolution of the request began",
+    )
+    request_start: float | None = Field(
+        default=None,
+        description="Unix timestamp when the backend began processing the request",
+    )
+    first_iteration: float | None = Field(
+        default=None,
+        description="Unix timestamp when the first iteration for a streaming began",
+    )
+    last_iteration: float | None = Field(
+        default=None,
+        description="Unix timestamp when the last iteration for a streaming completed",
+    )
+    iterations: int | None = Field(
+        default=None,
+        description="Total number of streaming update iterations performed",
+    )
+    request_end: float | None = Field(
+        default=None,
+        description="Unix timestamp when the backend completed processing the request",
+    )
+    resolve_end: float | None = Field(
+        default=None,
+        description="Unix timestamp when backend resolution of the request completed",
+    )
+    finalized: float | None = Field(
+        default=None,
+        description="Unix timestamp when request was processed by the scheduler",
+    )
+
+
+class RequestInfo(StandardBaseModel):
+    """
+    Complete information about a request in the scheduler system.
+
+    Encapsulates all metadata, status tracking, and timing information for requests
+    processed through the distributed scheduler. Provides comprehensive lifecycle
+    tracking from initial queuing through final completion, including error handling
+    and node identification for debugging and performance analysis.
+
+    Example:
+    ::
+        request = RequestInfo()
+        request.status = "in_progress"
+        start_time = request.started_at
+        completion_time = request.completed_at
+    """
+
+    request_id: str = Field(
+        description="Unique identifier for the request",
+        default_factory=lambda: str(uuid.uuid4()),
+    )
+    status: Literal[
+        "queued", "pending", "in_progress", "completed", "errored", "cancelled"
+    ] = Field(description="Current processing status of the request", default="queued")
+    scheduler_node_id: int = Field(
+        description="ID/rank of the scheduler node handling the request",
+        default=-1,
+    )
+    scheduler_process_id: int = Field(
+        description="ID/rank of the node's scheduler process handling the request",
+        default=-1,
+    )
+    scheduler_start_time: float = Field(
+        description="Unix timestamp when scheduler processing began",
+        default=-1,
+    )
+    timings: RequestTimings = Field(
+        default_factory=RequestTimings,
+        description="Timing measurements for the request lifecycle",
+    )
+
+    error: str | None = Field(
+        default=None, description="Error message if the request status is 'errored'"
+    )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def started_at(self) -> float | None:
+        """
+        Get the effective request processing start time.
+
+        :return: Unix timestamp when processing began, or None if not started
+        """
+        return self.timings.request_start or self.timings.resolve_start
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def completed_at(self) -> float | None:
+        """
+        Get the effective request processing completion time.
+
+        :return: Unix timestamp when processing completed, or None if not completed
+        """
+        return self.timings.request_end or self.timings.resolve_end
+
+    def model_copy(self, **_kwargs) -> RequestInfo:  # type: ignore[override]  # noqa: ARG002
+        """
+        Create a deep copy of the request info with copied timing objects.
+
+        :param kwargs: Additional keyword arguments for model copying
+        :return: New RequestInfo instance with independent timing objects
+        """
+        return super().model_copy(
+            update={
+                "timings": self.timings.model_copy(),
+            },
+            deep=False,
+        )

guidellm/schemas/request.py

@@ -0,0 +1,216 @@
+"""
+Request schema definitions for generation operations.
+
+Contains request models and data structures used to define and execute generation
+requests across different backend services. Provides standardized interfaces for
+request arguments, usage metrics tracking, and request type definitions that enable
+consistent interaction with various AI generation APIs.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Literal
+
+from pydantic import Field, computed_field
+
+from guidellm.utils import StandardBaseDict, StandardBaseModel
+
+__all__ = [
+    "GenerationRequest",
+    "GenerationRequestArguments",
+    "GenerativeRequestType",
+    "UsageMetrics",
+]
+
+
+GenerativeRequestType = Literal[
+    "text_completions",
+    "chat_completions",
+    "audio_transcriptions",
+    "audio_translations",
+]
+
+
+class GenerationRequestArguments(StandardBaseDict):
+    """
+    HTTP request arguments for generation operations.
+
+    Encapsulates all necessary HTTP request components including method, headers,
+    parameters, and payload data required to execute generation requests against
+    backend services. Supports file uploads and streaming responses.
+    """
+
+    method: str | None = Field(
+        default=None,
+        description="The HTTP method to use for the request (e.g., 'POST', 'GET').",
+    )
+    stream: bool | None = Field(
+        default=None,
+        description="Whether to stream the response, if applicable.",
+    )
+    headers: dict[str, str] | None = Field(
+        default=None,
+        description="Any headers to include in the request, if applicable.",
+    )
+    params: dict[str, Any] | None = Field(
+        default=None,
+        description="Query parameters to include in the request, if applicable.",
+    )
+    body: dict[str, Any] | None = Field(
+        default=None,
+        description="Content to include in the main request body.",
+    )
+    files: dict[str, Any] | None = Field(
+        default=None,
+        description="Files to include in the request, if applicable.",
+    )
+
+    def model_combine(
+        self, additional: GenerationRequestArguments | dict[str, Any]
+    ) -> GenerationRequestArguments:
+        """
+        Merge additional request arguments into the current instance.
+
+        Combines method and stream fields by overwriting, while merging collection
+        fields like headers, params, json_body, and files by extending existing values.
+
+        :param additional: Additional arguments to merge with current instance
+        :return: Updated instance with merged arguments
+        """
+        additional_dict = (
+            additional.model_dump()
+            if isinstance(additional, GenerationRequestArguments)
+            else additional
+        )
+
+        for overwrite in ("method", "stream"):
+            if (val := additional_dict.get(overwrite)) is not None:
+                setattr(self, overwrite, val)
+
+        for combine in ("headers", "params", "json_body", "files"):
+            if (val := additional_dict.get(combine)) is not None:
+                setattr(self, combine, {**getattr(self, combine, {}), **val})
+
+        return self
+
+
+class UsageMetrics(StandardBaseDict):
+    """
+    Multimodal usage metrics for generation requests.
+
+    Tracks resource consumption across different modalities including text, images,
+    video, and audio. Provides granular metrics for tokens, bytes, duration, and
+    format-specific measurements to enable comprehensive usage monitoring and billing.
+    """
+
+    # Text stats
+    text_tokens: int | None = Field(
+        default=None, description="Number of text tokens processed/generated."
+    )
+    text_words: int | None = Field(
+        default=None, description="Number of text words processed/generated."
+    )
+    text_characters: int | None = Field(
+        default=None, description="Number of text characters processed/generated."
+    )
+
+    # Vision image stats
+    image_tokens: int | None = Field(
+        default=None, description="Number of image tokens processed/generated."
+    )
+    image_count: int | None = Field(
+        default=None, description="Number of images processed/generated."
+    )
+    image_pixels: int | None = Field(
+        default=None, description="Number of image pixels processed/generated."
+    )
+    image_bytes: int | None = Field(
+        default=None, description="Number of image bytes processed/generated."
+    )
+
+    # Vision video stats
+    video_tokens: int | None = Field(
+        default=None, description="Number of video tokens processed/generated."
+    )
+    video_frames: int | None = Field(
+        default=None, description="Number of video frames processed/generated."
+    )
+    video_seconds: float | None = Field(
+        default=None, description="Duration of video processed/generated in seconds."
+    )
+    video_bytes: int | None = Field(
+        default=None, description="Number of video bytes processed/generated."
+    )
+
+    # Audio stats
+    audio_tokens: int | None = Field(
+        default=None, description="Number of audio tokens processed/generated."
+    )
+    audio_samples: int | None = Field(
+        default=None, description="Number of audio samples processed/generated."
+    )
+    audio_seconds: float | None = Field(
+        default=None, description="Duration of audio processed/generated in seconds."
+    )
+    audio_bytes: int | None = Field(
+        default=None, description="Number of audio bytes processed/generated."
+    )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_tokens(self) -> int | None:
+        """
+        Calculate total tokens across all modalities.
+
+        :return: Sum of text, image, video, and audio tokens, or None if all are None
+        """
+        return (self.text_tokens or 0) + (self.image_tokens or 0) + (
+            self.video_tokens or 0
+        ) + (self.audio_tokens or 0) or None
+
+
+class GenerationRequest(StandardBaseModel):
+    """
+    Complete request specification for backend generation operations.
+
+    Encapsulates all components needed to execute a generation request including
+    unique identification, request type specification, HTTP arguments, and input/output
+    usage metrics. Serves as the primary interface between the scheduler and backend
+    services for coordinating AI generation tasks.
+
+    Example::
+        request = GenerationRequest(
+            request_type="text_completions",
+            arguments=GenerationRequestArguments(
+                method="POST",
+                body={"prompt": "Hello world", "max_tokens": 100}
+            )
+        )
+    """
+
+    request_id: str = Field(
+        default_factory=lambda: str(uuid.uuid4()),
+        description="Unique identifier for the request.",
+    )
+    request_type: GenerativeRequestType | str = Field(
+        description=(
+            "Type of request. If url is not provided in arguments, "
+            "this will be used to determine the request url."
+        ),
+    )
+    arguments: GenerationRequestArguments = Field(
+        description=(
+            "Payload for the request, structured as a dictionary of arguments to pass "
+            "to the respective backend method. For example, can contain "
+            "'json', 'headers', 'files', etc."
+        )
+    )
+    input_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Input statistics including counts, sizes, and durations.",
+    )
+    output_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Output statistics including counts, sizes, and durations.",
+    )

guidellm/schemas/response.py

@@ -0,0 +1,119 @@
+"""
+Backend response models for request and response handling.
+
+Provides standardized response models for generation operations that capture
+output text, usage metrics, and compilation of request statistics. Ensures
+consistent data handling and statistics aggregation across different backend
+implementations.
+"""
+
+from __future__ import annotations
+
+from pydantic import Field
+
+from guidellm.schemas.info import RequestInfo
+from guidellm.schemas.request import GenerationRequest, UsageMetrics
+from guidellm.schemas.stats import GenerativeRequestStats
+from guidellm.utils import StandardBaseModel
+
+__all__ = ["GenerationResponse"]
+
+
+class GenerationResponse(StandardBaseModel):
+    """
+    Response model for backend generation operations.
+
+    Captures the output and metrics from a generation request, providing structured
+    data for text output, token usage statistics, and compilation of detailed
+    request statistics for analysis and monitoring purposes.
+
+    Example:
+    ::
+        response = GenerationResponse(
+            request_id="req-123",
+            text="Generated response text",
+            input_metrics=UsageMetrics(token_count=50),
+            output_metrics=UsageMetrics(token_count=25)
+        )
+        stats = response.compile_stats(request, info)
+    """
+
+    request_id: str = Field(
+        description="Unique identifier matching the original GenerationRequest."
+    )
+    request_args: str | None = Field(
+        description="Arguments passed to the backend for request processing."
+    )
+    text: str | None = Field(
+        default=None,
+        description="The generated response text.",
+    )
+    input_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Token usage statistics from the input prompt.",
+    )
+    output_metrics: UsageMetrics = Field(
+        default_factory=UsageMetrics,
+        description="Token usage statistics from the generated output.",
+    )
+
+    def compile_stats(
+        self,
+        request: GenerationRequest,
+        info: RequestInfo,
+        prefer_response: bool = True,
+    ) -> GenerativeRequestStats:
+        """
+        Compile and return comprehensive request statistics.
+
+        Merges metrics from the request and response objects to create a complete
+        statistical record, with preference given to response-level metrics when
+        available to ensure accuracy of actual execution data.
+
+        :param request: The original generation request containing input data
+        :param info: Metadata and timing information for the request execution
+        :param prefer_response: Whether to prefer response metrics over request
+            metrics when both are available
+        :return: A GenerativeRequestStats object containing detailed statistics
+        :raises ValueError: When request IDs don't match between objects
+        """
+        if request.request_id != self.request_id:
+            raise ValueError("Mismatched request IDs between request and response.")
+
+        if info.request_id != self.request_id:
+            raise ValueError("Mismatched request IDs between info and response.")
+
+        if info.status != "completed":
+            # clear out request output metrics if the request failed since
+            # those are not valid
+            request.output_metrics = UsageMetrics()
+
+        base_input = request.input_metrics if prefer_response else self.input_metrics
+        override_input = (
+            self.input_metrics if prefer_response else request.input_metrics
+        )
+        base_output = request.output_metrics if prefer_response else self.output_metrics
+        override_output = (
+            self.output_metrics if prefer_response else request.output_metrics
+        )
+
+        input_metrics_dict = base_input.model_dump()
+        for key, value in override_input.model_dump().items():
+            if value is not None:
+                input_metrics_dict[key] = value
+        output_metrics_dict = base_output.model_dump()
+        for key, value in override_output.model_dump().items():
+            if value is not None:
+                output_metrics_dict[key] = value
+
+        return GenerativeRequestStats(
+            request_id=self.request_id,
+            request_type=request.request_type,
+            request_args=str(
+                request.arguments.model_dump() if request.arguments else {}
+            ),
+            output=self.text,
+            info=info,
+            input_metrics=UsageMetrics(**input_metrics_dict),
+            output_metrics=UsageMetrics(**output_metrics_dict),
+        )