guidellm 0.4.0a21__py3-none-any.whl → 0.4.0a169__py3-none-any.whl

guidellm/dataset/synthetic.py

@@ -1,287 +0,0 @@
-import json
-import random
-from collections.abc import Iterable, Iterator
-from itertools import cycle
-from pathlib import Path
-from typing import Any, Literal, Optional, Union
-
-import yaml
-from datasets import (
-    Dataset,
-    DatasetDict,
-    IterableDataset,
-    IterableDatasetDict,
-)
-from pydantic import BaseModel, Field
-from transformers import PreTrainedTokenizerBase  # type: ignore[import]
-
-from guidellm.dataset.creator import ColumnInputTypes, DatasetCreator
-from guidellm.utils import EndlessTextCreator, IntegerRangeSampler, check_load_processor
-
-__all__ = [
-    "SyntheticDatasetConfig",
-    "SyntheticDatasetCreator",
-    "SyntheticTextItemsGenerator",
-]
-
-
-class SyntheticDatasetConfig(BaseModel):
-    prefix_tokens: int = Field(
-        description="The number of shared prefix tokens to prepend to each prompt.",
-        ge=0,
-        default=0,
-    )
-    prompt_tokens: int = Field(
-        description="The average number of text tokens generated for prompts.",
-        gt=0,
-    )
-    prompt_tokens_stdev: Optional[int] = Field(
-        description="The standard deviation of the tokens generated for prompts.",
-        gt=0,
-        default=None,
-    )
-    prompt_tokens_min: Optional[int] = Field(
-        description="The minimum number of text tokens generated for prompts.",
-        gt=0,
-        default=None,
-    )
-    prompt_tokens_max: Optional[int] = Field(
-        description="The maximum number of text tokens generated for prompts.",
-        gt=0,
-        default=None,
-    )
-    output_tokens: int = Field(
-        description="The average number of text tokens generated for outputs.",
-        gt=0,
-    )
-    output_tokens_stdev: Optional[int] = Field(
-        description="The standard deviation of the tokens generated for outputs.",
-        gt=0,
-        default=None,
-    )
-    output_tokens_min: Optional[int] = Field(
-        description="The minimum number of text tokens generated for outputs.",
-        gt=0,
-        default=None,
-    )
-    output_tokens_max: Optional[int] = Field(
-        description="The maximum number of text tokens generated for outputs.",
-        gt=0,
-        default=None,
-    )
-    samples: int = Field(
-        description="The number of samples to generate for the dataset.",
-        gt=0,
-        default=1000,
-    )
-    source: str = Field(
-        description="The source of the text data to be used for generation.",
-        default="data:prideandprejudice.txt.gz",
-    )
-
-    @staticmethod
-    def parse_str(data: Union[str, Path]) -> "SyntheticDatasetConfig":
-        if (
-            isinstance(data, Path)
-            or data.strip().endswith(".config")
-            or data.strip().endswith(".yaml")
-        ):
-            return SyntheticDatasetConfig.parse_config_file(data)
-
-        if data.strip().startswith("{"):
-            return SyntheticDatasetConfig.parse_json(data)
-
-        if data.count("=") > 1:
-            return SyntheticDatasetConfig.parse_key_value_pairs(data)
-
-        raise ValueError(
-            f"Unsupported data format. Expected JSON or key-value pairs, got {data}"
-        )
-
-    @staticmethod
-    def parse_json(data: str) -> "SyntheticDatasetConfig":
-        config_dict = json.loads(data.strip())
-
-        return SyntheticDatasetConfig(**config_dict)
-
-    @staticmethod
-    def parse_key_value_pairs(data: str) -> "SyntheticDatasetConfig":
-        config_dict = {}
-        items = data.strip().split(",")
-        for item in items:
-            key, value = item.split("=")
-            config_dict[key.strip()] = (
-                int(value.strip()) if value.strip().isnumeric() else value.strip()
-            )
-
-        return SyntheticDatasetConfig(**config_dict)  # type: ignore[arg-type]
-
-    @staticmethod
-    def parse_config_file(data: Union[str, Path]) -> "SyntheticDatasetConfig":
-        with Path(data).open("r") as file:
-            config_dict = yaml.safe_load(file)
-
-        return SyntheticDatasetConfig(**config_dict)
-
-
-class SyntheticTextItemsGenerator(
-    Iterable[
-        dict[
-            Literal["prompt", "prompt_tokens_count", "output_tokens_count"],
-            Union[str, int],
-        ]
-    ]
-):
-    def __init__(
-        self,
-        config: SyntheticDatasetConfig,
-        processor: PreTrainedTokenizerBase,
-        random_seed: int,
-    ):
-        self.config = config
-        self.processor = processor
-        self.random_seed = random_seed
-        self.text_creator = EndlessTextCreator(
-            data=config.source,
-        )
-
-    def __iter__(
-        self,
-    ) -> Iterator[
-        dict[
-            Literal["prompt", "prompt_tokens_count", "output_tokens_count"],
-            Union[str, int],
-        ]
-    ]:
-        prompt_tokens_sampler = IntegerRangeSampler(
-            average=self.config.prompt_tokens,
-            variance=self.config.prompt_tokens_stdev,
-            min_value=self.config.prompt_tokens_min,
-            max_value=self.config.prompt_tokens_max,
-            random_seed=self.random_seed,
-        )
-        output_tokens_sampler = IntegerRangeSampler(
-            average=self.config.output_tokens,
-            variance=self.config.output_tokens_stdev,
-            min_value=self.config.output_tokens_min,
-            max_value=self.config.output_tokens_max,
-            random_seed=self.random_seed + 1,  # ensure diff dist from prompts
-        )
-        # ensure diff distribution from output tokens
-        rand = random.Random(self.random_seed + 2)  # noqa: S311
-        unique_prefix_iter = cycle(self.processor.get_vocab().values())
-
-        prefix_index = rand.randint(0, len(self.text_creator.words))
-        prefix_tokens = self._create_prompt(self.config.prefix_tokens, prefix_index)
-
-        for _, prompt_tokens, output_tokens in zip(
-            range(self.config.samples),
-            prompt_tokens_sampler,
-            output_tokens_sampler,
-        ):
-            start_index = rand.randint(0, len(self.text_creator.words))
-            prompt_text = self.processor.decode(
-                prefix_tokens
-                + self._create_prompt(
-                    prompt_tokens, start_index, next(unique_prefix_iter)
-                ),
-                skip_special_tokens=True,
-            )
-            yield {
-                "prompt": prompt_text,
-                "prompt_tokens_count": self.config.prefix_tokens + prompt_tokens,
-                "output_tokens_count": output_tokens,
-            }
-
-    def _create_prompt(
-        self, prompt_tokens: int, start_index: int, unique_prefix: Optional[int] = None
-    ) -> list[int]:
-        if prompt_tokens <= 0:
-            return []
-
-        left = start_index
-        right = start_index + 4 * prompt_tokens
-        start_tokens = [unique_prefix] if unique_prefix else []
-
-        while left < right:
-            mid = (left + right) // 2
-            test_prompt = self.text_creator.create_text(start_index, mid - start_index)
-            test_tokens = start_tokens + self.processor.encode(test_prompt)
-
-            if len(test_tokens) == prompt_tokens:
-                return test_tokens
-            elif len(test_tokens) < prompt_tokens:
-                left = mid + 1
-            else:
-                right = mid
-
-        final_text = self.text_creator.create_text(start_index, left - start_index)
-        return start_tokens + self.processor.encode(final_text)
-
-
-class SyntheticDatasetCreator(DatasetCreator):
-    @classmethod
-    def is_supported(
-        cls,
-        data: Any,
-        data_args: Optional[dict[str, Any]],  # noqa: ARG003
-    ) -> bool:
-        if (
-            isinstance(data, Path)
-            and data.exists()
-            and data.suffix in {".config", ".yaml"}
-        ):
-            return True
-
-        if isinstance(data, str):
-            data_str: str = data.strip()
-            if (
-                data_str.startswith("{")
-                or data_str.count("=") > 1
-                or data_str.endswith((".config", ".yaml"))
-            ):
-                return True
-
-        return False
-
-    @classmethod
-    def handle_create(
-        cls,
-        data: Any,
-        data_args: Optional[dict[str, Any]],
-        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
-        processor_args: Optional[dict[str, Any]],
-        random_seed: int,
-    ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
-        processor = check_load_processor(
-            processor,
-            processor_args,
-            error_msg=(
-                "Processor/tokenizer required for synthetic dataset generation."
-            ),
-        )
-
-        config = SyntheticDatasetConfig.parse_str(data)
-        generator = SyntheticTextItemsGenerator(config, processor, random_seed)
-        items = list(generator)
-
-        return Dataset.from_list(items, **(data_args or {}))
-
-    @classmethod
-    def extract_args_column_mappings(
-        cls,
-        data_args: Optional[dict[str, Any]],
-    ) -> dict[ColumnInputTypes, str]:
-        data_args_columns = super().extract_args_column_mappings(data_args)
-
-        if data_args_columns:
-            raise ValueError(
-                f"Column mappings are not supported for synthetic datasets. "
-                f"Got {data_args_columns}"
-            )
-
-        return {
-            "prompt_column": "prompt",
-            "prompt_tokens_count_column": "prompt_tokens_count",
-            "output_tokens_count_column": "output_tokens_count",
-        }

guidellm/objects/__init__.py

@@ -1,18 +0,0 @@
-from .pydantic import StandardBaseModel, StatusBreakdown
-from .statistics import (
-    DistributionSummary,
-    Percentiles,
-    RunningStats,
-    StatusDistributionSummary,
-    TimeRunningStats,
-)
-
-__all__ = [
-    "DistributionSummary",
-    "Percentiles",
-    "RunningStats",
-    "StandardBaseModel",
-    "StatusBreakdown",
-    "StatusDistributionSummary",
-    "TimeRunningStats",
-]

guidellm/objects/pydantic.py

@@ -1,89 +0,0 @@
-import json
-from pathlib import Path
-from typing import Any, Generic, Optional, TypeVar
-
-import yaml
-from loguru import logger
-from pydantic import BaseModel, ConfigDict, Field
-
-__all__ = ["StandardBaseModel", "StatusBreakdown"]
-
-T = TypeVar("T", bound="StandardBaseModel")
-
-
-class StandardBaseModel(BaseModel):
-    """
-    A base class for Pydantic models throughout GuideLLM enabling standard
-    configuration and logging.
-    """
-
-    model_config = ConfigDict(
-        extra="ignore",
-        use_enum_values=True,
-        validate_assignment=True,
-        from_attributes=True,
-    )
-
-    def __init__(self, /, **data: Any) -> None:
-        super().__init__(**data)
-        logger.debug(
-            "Initialized new instance of {} with data: {}",
-            self.__class__.__name__,
-            data,
-        )
-
-    @classmethod
-    def get_default(cls: type[T], field: str) -> Any:
-        """Get default values for model fields"""
-        return cls.model_fields[field].default
-
-    @classmethod
-    def from_file(cls: type[T], filename: Path, overrides: Optional[dict] = None) -> T:
-        """
-        Attempt to create a new instance of the model using
-        data loaded from json or yaml file.
-        """
-        try:
-            with filename.open() as f:
-                if str(filename).endswith(".json"):
-                    data = json.load(f)
-                else:  # Assume everything else is yaml
-                    data = yaml.safe_load(f)
-        except (json.JSONDecodeError, yaml.YAMLError) as e:
-            logger.error(f"Failed to parse {filename} as type {cls.__name__}")
-            raise ValueError(f"Error when parsing file: {filename}") from e
-
-        data.update(overrides)
-        return cls.model_validate(data)
-
-
-SuccessfulT = TypeVar("SuccessfulT")
-ErroredT = TypeVar("ErroredT")
-IncompleteT = TypeVar("IncompleteT")
-TotalT = TypeVar("TotalT")
-
-
-class StatusBreakdown(BaseModel, Generic[SuccessfulT, ErroredT, IncompleteT, TotalT]):
-    """
-    A base class for Pydantic models that are separated by statuses including
-    successful, incomplete, and errored. It additionally enables the inclusion
-    of total, which is intended as the combination of all statuses.
-    Total may or may not be used depending on if it duplicates information.
-    """
-
-    successful: SuccessfulT = Field(
-        description="The results with a successful status.",
-        default=None,  # type: ignore[assignment]
-    )
-    errored: ErroredT = Field(
-        description="The results with an errored status.",
-        default=None,  # type: ignore[assignment]
-    )
-    incomplete: IncompleteT = Field(
-        description="The results with an incomplete status.",
-        default=None,  # type: ignore[assignment]
-    )
-    total: TotalT = Field(
-        description="The combination of all statuses.",
-        default=None,  # type: ignore[assignment]
-    )

guidellm/request/__init__.py

@@ -1,18 +0,0 @@
-from .loader import (
-    GenerativeRequestLoader,
-    GenerativeRequestLoaderDescription,
-    RequestLoader,
-    RequestLoaderDescription,
-)
-from .request import GenerationRequest
-from .types import RequestT, ResponseT
-
-__all__ = [
-    "GenerationRequest",
-    "GenerativeRequestLoader",
-    "GenerativeRequestLoaderDescription",
-    "RequestLoader",
-    "RequestLoaderDescription",
-    "RequestT",
-    "ResponseT",
-]

guidellm/request/loader.py

@@ -1,284 +0,0 @@
-from abc import abstractmethod
-from collections.abc import Iterable, Iterator
-from pathlib import Path
-from typing import (
-    Any,
-    Literal,
-    Optional,
-    Union,
-)
-
-from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
-from transformers import PreTrainedTokenizerBase  # type: ignore[import]
-
-from guidellm.config import settings
-from guidellm.dataset import ColumnInputTypes, load_dataset
-from guidellm.objects import StandardBaseModel
-from guidellm.request.request import GenerationRequest
-
-__all__ = [
-    "GenerativeRequestLoader",
-    "GenerativeRequestLoaderDescription",
-    "RequestLoader",
-    "RequestLoaderDescription",
-]
-
-
-class RequestLoaderDescription(StandardBaseModel):
-    type_: Literal["request_loader"] = "request_loader"
-
-
-class RequestLoader(Iterable):
-    @abstractmethod
-    def __iter__(self) -> Iterator: ...
-
-    @abstractmethod
-    def __len__(self) -> int: ...
-
-    @property
-    @abstractmethod
-    def description(self) -> RequestLoaderDescription: ...
-
-
-class GenerativeRequestLoaderDescription(RequestLoaderDescription):
-    type_: Literal["generative_request_loader"] = "generative_request_loader"  # type: ignore[assignment]
-    data: str
-    data_args: Optional[dict[str, Any]]
-    processor: str
-    processor_args: Optional[dict[str, Any]]
-
-
-class GenerativeRequestLoader(RequestLoader):
-    DEFAULT_PROMPT_COLUMNS = [
-        "prompt",
-        "prompts",
-        "instruction",
-        "instructions",
-        "question",
-        "questions",
-        "input",
-        "inputs",
-        "context",
-        "content",
-        "conversation",
-        "conversations",
-        "turn",
-        "turns",
-        "text",
-    ]
-
-    def __init__(
-        self,
-        data: Union[
-            str,
-            Path,
-            Iterable[Union[str, dict[str, Any]]],
-            Dataset,
-            DatasetDict,
-            IterableDataset,
-            IterableDatasetDict,
-        ],
-        data_args: Optional[dict[str, Any]],
-        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
-        processor_args: Optional[dict[str, Any]],
-        shuffle: bool = True,
-        iter_type: Literal["finite", "infinite"] = "finite",
-        random_seed: int = 42,
-    ):
-        self.data = data
-        self.data_args = data_args
-        dataset, args_column_mappings = load_dataset(
-            data,
-            data_args,
-            processor,
-            processor_args,
-            random_seed,
-        )
-        self.dataset = dataset
-        self.processor = processor
-        self.processor_args = processor_args
-        self.shuffle = shuffle
-        self.iter_type = iter_type
-        self.random_seed = random_seed
-
-        self.column_mappings = self._create_column_mappings(args_column_mappings)
-        self.preserve_iter_state = iter_type == "infinite"  # ensure no caching requests
-        self._preserved_iter = None
-
-    def __iter__(self) -> Iterator[GenerationRequest]:
-        scope_create_count = 0
-
-        while (dataset_iter := self._get_dataset_iter(scope_create_count)) is not None:
-            scope_create_count += 1
-
-            for item in dataset_iter:
-                yield self._create_request(item)
-
-            self._preserved_iter = None
-
-    def __len__(self) -> int:
-        if self.iter_type == "finite":
-            return self.num_unique_items()
-
-        raise ValueError(f"Unable to determine length of dataset: {self.data}")
-
-    @property
-    def description(self) -> GenerativeRequestLoaderDescription:
-        return GenerativeRequestLoaderDescription(
-            data=str(self.data),
-            data_args=self.data_args,
-            processor=str(self.processor),
-            processor_args=self.processor_args,
-        )
-
-    def num_unique_items(self, raise_err: bool = True) -> int:
-        try:
-            return len(self.dataset)
-        except Exception:  # noqa: BLE001, S110
-            pass
-
-        dataset_size = self.dataset.info.dataset_size
-        if dataset_size is not None:
-            return dataset_size
-
-        if raise_err:
-            raise ValueError("Unable to determine number of items in the dataset")
-
-        return -1
-
-    def _create_column_mappings(
-        self,
-        args_column_mappings: dict[ColumnInputTypes, str],
-    ) -> dict[ColumnInputTypes, str]:
-        column_mappings: dict[ColumnInputTypes, str] = {}
-
-        if "text_column" in args_column_mappings:
-            column_mappings["prompt_column"] = args_column_mappings["text_column"]
-        else:
-            column_mappings["prompt_column"] = self._extract_text_column()
-
-        if "prompt_tokens_count_column" in args_column_mappings:
-            column_mappings["prompt_tokens_count_column"] = args_column_mappings[
-                "prompt_tokens_count_column"
-            ]
-        elif prompt_tokens_count_column := self._extract_prompt_tokens_count_column():
-            column_mappings["prompt_tokens_count_column"] = prompt_tokens_count_column
-
-        if "output_tokens_count_column" in args_column_mappings:
-            column_mappings["output_tokens_count_column"] = args_column_mappings[
-                "output_tokens_count_column"
-            ]
-        elif output_tokens_count_column := self._extract_output_tokens_count_column():
-            column_mappings["output_tokens_count_column"] = output_tokens_count_column
-
-        return column_mappings
-
-    def _extract_text_column(self) -> str:
-        column_names = self._dataset_columns(
-            err_msg=(
-                "Unable to determine text column from dataset and it is required. "
-                "To specify the text column, set the 'text_column' key in the "
-                "'data_args' dictionary."
-            )
-        )
-
-        if not column_names:
-            raise ValueError(
-                "Unable to determine text column from dataset and it is required. "
-                "To specify the text column, set the 'text_column' key in the "
-                "'data_args' dictionary."
-            )
-
-        if len(column_names) == 1:
-            return column_names[0]
-
-        for def_column in self.DEFAULT_PROMPT_COLUMNS:
-            if def_column in column_names:
-                return def_column
-
-        raise ValueError(
-            f"Unable to determine text column from dataset columns: {column_names}. "
-            "To specify the text column, set the 'text_column' key in the "
-            "'data_args' dictionary."
-        )
-
-    def _extract_prompt_tokens_count_column(self) -> Optional[str]:
-        column_names = self._dataset_columns()
-
-        if column_names and "prompt_tokens_count" in column_names:
-            return "prompt_tokens_count"
-
-        if column_names and "prompt_tokens" in column_names:
-            return "prompt_tokens"
-
-        return None
-
-    def _extract_output_tokens_count_column(self) -> Optional[str]:
-        column_names = self._dataset_columns()
-
-        if column_names and "output_tokens_count" in column_names:
-            return "output_tokens_count"
-
-        if column_names and "output_tokens" in column_names:
-            return "output_tokens"
-
-        return None
-
-    def _dataset_columns(self, err_msg: Optional[str] = None) -> Optional[list[str]]:
-        try:
-            column_names = self.dataset.column_names
-
-            if not column_names and err_msg:
-                raise ValueError(f"No column names found in dataset: {self.data}")
-        except Exception as err:
-            if err_msg:
-                raise ValueError(err_msg) from err
-
-            column_names = None
-
-        return column_names
-
-    def _get_dataset_iter(
-        self, scope_create_count: int
-    ) -> Optional[Iterator[dict[str, Any]]]:
-        if scope_create_count > 0 and self.iter_type != "infinite":
-            return None
-
-        if self.preserve_iter_state and self._preserved_iter is not None:
-            return self._preserved_iter
-
-        dataset = (
-            self.dataset
-            if not self.shuffle
-            else self.dataset.shuffle(seed=self.random_seed)
-        )
-
-        dataset_iter = iter(dataset)
-
-        if self.preserve_iter_state:
-            self._preserved_iter = dataset_iter
-
-        return dataset_iter
-
-    def _create_request(self, item: dict[str, Any]) -> GenerationRequest:
-        prompt_tokens = (
-            item[self.column_mappings["prompt_tokens_count_column"]]
-            if "prompt_tokens_count_column" in self.column_mappings
-            else None
-        )
-        output_tokens = (
-            item[self.column_mappings["output_tokens_count_column"]]
-            if "output_tokens_count_column" in self.column_mappings
-            else None
-        )
-
-        return GenerationRequest(
-            request_type=settings.preferred_route,
-            content=item[self.column_mappings["prompt_column"]],
-            stats=(
-                {"prompt_tokens": prompt_tokens} if prompt_tokens is not None else {}
-            ),
-            constraints=(
-                {"output_tokens": output_tokens} if output_tokens is not None else {}
-            ),
-        )