guidellm 0.1.0__py3-none-any.whl → 0.2.0rc20250418__py3-none-any.whl

guidellm/config.py

@@ -1,18 +1,17 @@
 import json
+from collections.abc import Sequence
 from enum import Enum
-from typing import Dict, List, Optional, Sequence
+from typing import Literal, Optional
 
 from pydantic import BaseModel, Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 __all__ = [
     "DatasetSettings",
-    "EmulatedDataSettings",
     "Environment",
     "LoggingSettings",
     "OpenAISettings",
     "print_config",
-    "ReportGenerationSettings",
     "Settings",
     "reload_settings",
     "settings",
@@ -55,7 +54,7 @@ class DatasetSettings(BaseModel):
     Dataset settings for the application
     """
 
-    preferred_data_columns: List[str] = Field(
+    preferred_data_columns: list[str] = Field(
         default_factory=lambda: [
             "prompt",
             "instruction",
@@ -69,53 +68,23 @@ class DatasetSettings(BaseModel):
             "data",
         ]
     )
-    preferred_data_splits: List[str] = Field(
+    preferred_data_splits: list[str] = Field(
         default_factory=lambda: ["test", "tst", "validation", "val", "train"]
     )
 
 
-class EmulatedDataSettings(BaseModel):
-    """
-    Emulated data settings for the application to use
-    """
-
-    source: str = "https://www.gutenberg.org/files/1342/1342-0.txt"
-    filter_start: str = "It is a truth universally acknowledged, that a"
-    filter_end: str = "CHISWICK PRESS:--CHARLES WHITTINGHAM AND CO."
-    clean_text_args: Dict[str, bool] = Field(
-        default_factory=lambda: {
-            "fix_encoding": True,
-            "clean_whitespace": True,
-            "remove_empty_lines": True,
-            "force_new_line_punctuation": True,
-        }
-    )
-
-
 class OpenAISettings(BaseModel):
     """
     OpenAI settings for the application to connect to the API
     for OpenAI server based pathways
     """
 
-    # OpenAI API key.
-    api_key: str = "invalid_token"
-
-    # OpenAI-compatible server URL
-    # NOTE: The default value is default address of llama.cpp web server
-    base_url: str = "http://localhost:8000/v1"
-
-    max_gen_tokens: int = 4096
-
-
-class ReportGenerationSettings(BaseModel):
-    """
-    Report generation settings for the application
-    """
-
-    source: str = ""
-    report_html_match: str = "window.report_data = {};"
-    report_html_placeholder: str = "{}"
+    api_key: Optional[str] = None
+    bearer_token: Optional[str] = None
+    organization: Optional[str] = None
+    project: Optional[str] = None
+    base_url: str = "http://localhost:8000"
+    max_output_tokens: int = 16384
 
 
 class Settings(BaseSettings):
@@ -141,27 +110,40 @@ class Settings(BaseSettings):
 
     # general settings
     env: Environment = Environment.PROD
-    request_timeout: int = 30
-    max_concurrency: int = 512
-    num_sweep_profiles: int = 9
+    default_async_loop_sleep: float = 10e-5
     logging: LoggingSettings = LoggingSettings()
+    default_sweep_number: int = 10
+
+    # HTTP settings
+    request_timeout: int = 60 * 5  # 5 minutes
+    request_http2: bool = True
+
+    # Scheduler settings
+    max_concurrency: int = 512
+    max_worker_processes: int = 10
+    max_add_requests_per_loop: int = 20
 
     # Data settings
     dataset: DatasetSettings = DatasetSettings()
-    emulated_data: EmulatedDataSettings = EmulatedDataSettings()
 
-    # Request settings
+    # Request/stats settings
+    preferred_prompt_tokens_source: Optional[
+        Literal["request", "response", "local"]
+    ] = "response"
+    preferred_output_tokens_source: Optional[
+        Literal["request", "response", "local"]
+    ] = "response"
+    preferred_backend: Literal["openai"] = "openai"
     openai: OpenAISettings = OpenAISettings()
 
-    # Report settings
-    report_generation: ReportGenerationSettings = ReportGenerationSettings()
+    # Output settings
+    table_border_char: str = "="
+    table_headers_border_char: str = "-"
+    table_column_separator_char: str = "|"
 
     @model_validator(mode="after")
     @classmethod
     def set_default_source(cls, values):
-        if not values.report_generation.source:
-            values.report_generation.source = ENV_REPORT_MAPPING.get(values.env)
-
         return values
 
     def generate_env_file(self) -> str:
@@ -197,7 +179,7 @@ class Settings(BaseSettings):
                 if isinstance(sub_value, Sequence) and not isinstance(sub_value, str):
                     value_str = ",".join(f'"{item}"' for item in sub_value)
                     env_file += f"{tag}=[{value_str}]\n"
-                elif isinstance(sub_value, Dict):
+                elif isinstance(sub_value, dict):
                     value_str = json.dumps(sub_value)
                     env_file += f"{tag}={value_str}\n"
                 elif not sub_value:
@@ -228,7 +210,3 @@ def print_config():
     Print the current configuration settings
     """
     print(f"Settings: \n{settings.generate_env_file()}")  # noqa: T201
-
-
-if __name__ == "__main__":
-    print_config()

guidellm/data/__init__.py

@@ -0,0 +1,4 @@
+"""
+Required for python < 3.12
+https://docs.python.org/3/library/importlib.resources.html#importlib.resources.files
+"""

guidellm/dataset/__init__.py

@@ -0,0 +1,22 @@
+from .creator import ColumnInputTypes, DatasetCreator
+from .entrypoints import load_dataset
+from .file import FileDatasetCreator
+from .hf_datasets import HFDatasetsCreator
+from .in_memory import InMemoryDatasetCreator
+from .synthetic import (
+    SyntheticDatasetConfig,
+    SyntheticDatasetCreator,
+    SyntheticTextItemsGenerator,
+)
+
+__all__ = [
+    "DatasetCreator",
+    "ColumnInputTypes",
+    "HFDatasetsCreator",
+    "load_dataset",
+    "FileDatasetCreator",
+    "InMemoryDatasetCreator",
+    "SyntheticDatasetCreator",
+    "SyntheticDatasetConfig",
+    "SyntheticTextItemsGenerator",
+]

guidellm/dataset/creator.py

@@ -0,0 +1,213 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Literal, Optional, Union
+
+from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
+from transformers import PreTrainedTokenizerBase  # type: ignore[import]
+
+__all__ = ["DatasetCreator", "ColumnInputTypes"]
+
+ColumnInputTypes = Literal[
+    "prompt_column",
+    "text_column",
+    "prompt_tokens_count_column",
+    "output_tokens_count_column",
+]
+
+
+class DatasetCreator(ABC):
+    DEFAULT_SPLITS_TRAIN = [
+        "train",
+        "training",
+        "train_set",
+        "training_set",
+        "train_dataset",
+        "training_dataset",
+        "train_data",
+        "training_data",
+        "pretrain",
+        "pretrain_set",
+        "pretrain_dataset",
+        "pretrain_data",
+        "pretraining",
+    ]
+    DEFAULT_SPLITS_CALIB = [
+        "calibration",
+        "calib",
+        "cal",
+        "calibration_set",
+        "calib_set",
+        "cal_set",
+        "calibration_dataset",
+        "calib_dataset",
+        "cal_set",
+        "calibration_data",
+        "calib_data",
+        "cal_data",
+    ]
+    DEFAULT_SPLITS_VAL = [
+        "validation",
+        "val",
+        "valid",
+        "validation_set",
+        "val_set",
+        "validation_dataset",
+        "val_dataset",
+        "validation_data",
+        "val_data",
+        "dev",
+        "dev_set",
+        "dev_dataset",
+        "dev_data",
+    ]
+    DEFAULT_SPLITS_TEST = [
+        "test",
+        "testing",
+        "test_set",
+        "testing_set",
+        "test_dataset",
+        "testing_dataset",
+        "test_data",
+        "testing_data",
+        "eval",
+        "eval_set",
+        "eval_dataset",
+        "eval_data",
+    ]
+    DEFAULT_SPLITS_DATASET: dict[str, str] = {}
+
+    @classmethod
+    def create(
+        cls,
+        data: Any,
+        data_args: Optional[dict[str, Any]],
+        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
+        processor_args: Optional[dict[str, Any]],
+        random_seed: int = 42,
+        split_pref_order: Optional[list[str]] = None,
+    ) -> tuple[Union[Dataset, IterableDataset], dict[ColumnInputTypes, str]]:
+        if not cls.is_supported(data, data_args):
+            raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ")
+
+        split = cls.extract_args_split(data_args)
+        column_mappings = cls.extract_args_column_mappings(data_args)
+        dataset = cls.handle_create(
+            data, data_args, processor, processor_args, random_seed
+        )
+
+        if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+            dataset = cls.extract_dataset_split(dataset, split, split_pref_order)
+
+        if not isinstance(dataset, (Dataset, IterableDataset)):
+            raise ValueError(
+                f"Unsupported data type: {type(dataset)} given for {dataset}."
+            )
+
+        return dataset, column_mappings
+
+    @classmethod
+    def extract_args_split(cls, data_args: Optional[dict[str, Any]]) -> str:
+        split = "auto"
+
+        if data_args and "split" in data_args:
+            split = data_args["split"]
+            del data_args["split"]
+
+        return split
+
+    @classmethod
+    def extract_args_column_mappings(
+        cls,
+        data_args: Optional[dict[str, Any]],
+    ) -> dict[ColumnInputTypes, str]:
+        columns: dict[ColumnInputTypes, str] = {}
+
+        if data_args:
+            if "prompt_column" in data_args:
+                columns["prompt_column"] = data_args["prompt_column"]
+                del data_args["prompt_column"]
+
+            if "prompt_tokens_count_column" in data_args:
+                columns["prompt_tokens_count_column"] = data_args[
+                    "prompt_tokens_count_column"
+                ]
+                del data_args["prompt_tokens_count_column"]
+
+            if "output_tokens_count_column" in data_args:
+                columns["output_tokens_count_column"] = data_args[
+                    "output_tokens_count_column"
+                ]
+                del data_args["output_tokens_count_column"]
+
+        return columns
+
+    @classmethod
+    def extract_dataset_name(
+        cls, dataset: Union[Dataset, IterableDataset, DatasetDict, IterableDatasetDict]
+    ) -> Optional[str]:
+        if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+            dataset = dataset[list(dataset.keys())[0]]
+
+        if isinstance(dataset, (Dataset, IterableDataset)):
+            if not hasattr(dataset, "info") or not hasattr(
+                dataset.info, "dataset_name"
+            ):
+                return None
+
+            return dataset.info.dataset_name
+
+        raise ValueError(f"Unsupported data type: {type(dataset)} given for {dataset}.")
+
+    @classmethod
+    def extract_dataset_split(
+        cls,
+        dataset: Union[DatasetDict, IterableDatasetDict],
+        specified_split: Union[Literal["auto"], str] = "auto",
+        split_pref_order: Optional[Union[Literal["auto"], list[str]]] = "auto",
+    ) -> Union[Dataset, IterableDataset]:
+        if not isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+            raise ValueError(
+                f"Unsupported data type: {type(dataset)} given for {dataset}."
+            )
+
+        if specified_split != "auto":
+            if specified_split not in dataset:
+                raise ValueError(
+                    f"Split {specified_split} not found in dataset {dataset}."
+                )
+
+            return dataset[specified_split]
+
+        dataset_name = cls.extract_dataset_name(dataset)
+
+        if dataset_name and dataset_name in cls.DEFAULT_SPLITS_DATASET:
+            return dataset[cls.DEFAULT_SPLITS_DATASET[dataset_name]]
+
+        if split_pref_order == "auto":
+            split_pref_order = [
+                *cls.DEFAULT_SPLITS_TEST,
+                *cls.DEFAULT_SPLITS_VAL,
+                *cls.DEFAULT_SPLITS_CALIB,
+                *cls.DEFAULT_SPLITS_TRAIN,
+            ]
+
+        for test_split in split_pref_order or []:
+            if test_split in dataset:
+                return dataset[test_split]
+
+        return dataset[list(dataset.keys())[0]]
+
+    @classmethod
+    @abstractmethod
+    def is_supported(cls, data: Any, data_args: Optional[dict[str, Any]]) -> bool: ...
+
+    @classmethod
+    @abstractmethod
+    def handle_create(
+        cls,
+        data: Any,
+        data_args: Optional[dict[str, Any]],
+        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
+        processor_args: Optional[dict[str, Any]],
+        random_seed: int,
+    ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: ...

guidellm/dataset/entrypoints.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from datasets import Dataset, IterableDataset
+from transformers import PreTrainedTokenizerBase  # type: ignore[import]
+
+from guidellm.dataset.creator import ColumnInputTypes
+from guidellm.dataset.file import FileDatasetCreator
+from guidellm.dataset.hf_datasets import HFDatasetsCreator
+from guidellm.dataset.in_memory import InMemoryDatasetCreator
+from guidellm.dataset.synthetic import SyntheticDatasetCreator
+
+__all__ = ["load_dataset"]
+
+
+def load_dataset(
+    data: Any,
+    data_args: Optional[dict[str, Any]],
+    processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
+    processor_args: Optional[dict[str, Any]],
+    random_seed: int = 42,
+    split_pref_order: Optional[list[str]] = None,
+) -> tuple[Union[Dataset, IterableDataset], dict[ColumnInputTypes, str]]:
+    creators = [
+        InMemoryDatasetCreator,
+        SyntheticDatasetCreator,
+        FileDatasetCreator,
+        HFDatasetsCreator,
+    ]
+
+    for creator in creators:
+        if creator.is_supported(data, data_args):
+            return creator.create(
+                data,
+                data_args,
+                processor,
+                processor_args,
+                random_seed,
+                split_pref_order,
+            )
+
+    raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ")

guidellm/dataset/file.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import pandas as pd  # type: ignore[import]
+from datasets import (
+    Dataset,
+    DatasetDict,
+    IterableDataset,
+    IterableDatasetDict,
+    load_dataset,
+)
+from transformers import PreTrainedTokenizerBase  # type: ignore[import]
+
+from guidellm.dataset.creator import DatasetCreator
+
+__all__ = ["FileDatasetCreator"]
+
+
+class FileDatasetCreator(DatasetCreator):
+    SUPPORTED_TYPES = {
+        ".txt",
+        ".text",
+        ".csv",
+        ".json",
+        ".jsonl",
+        ".parquet",
+        ".arrow",
+        ".hdf5",
+        ".tar",
+    }
+
+    @classmethod
+    def is_supported(cls, data: Any, data_args: Optional[dict[str, Any]]) -> bool:  # noqa: ARG003
+        if isinstance(data, (str, Path)) and (path := Path(data)).exists():
+            # local folder or py file, assume supported
+            return path.suffix.lower() in cls.SUPPORTED_TYPES
+
+        return False
+
+    @classmethod
+    def handle_create(
+        cls,
+        data: Any,
+        data_args: Optional[dict[str, Any]],
+        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],  # noqa: ARG003
+        processor_args: Optional[dict[str, Any]],  # noqa: ARG003
+        random_seed: int,  # noqa: ARG003
+    ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
+        if not isinstance(data, (str, Path)):
+            raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ")
+
+        path = Path(data)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        if not path.is_file():
+            raise ValueError(f"Unsupported data type: {path} given for {path}. ")
+
+        if path.suffix.lower() not in cls.SUPPORTED_TYPES:
+            raise ValueError(f"Unsupported file type: {path.suffix} given for {path}. ")
+
+        return cls.load_dataset(path, data_args)
+
+    @classmethod
+    def load_dataset(
+        cls, path: Path, data_args: Optional[dict[str, Any]]
+    ) -> Union[Dataset, IterableDataset]:
+        if path.suffix.lower() in {".txt", ".text"}:
+            with path.open("r") as file:
+                items = file.readlines()
+
+            dataset = Dataset.from_dict({"text": items}, **(data_args or {}))
+        elif path.suffix.lower() == ".csv":
+            dataset = load_dataset("csv", data_files=path, **(data_args or {}))
+        elif path.suffix.lower() in {".json", ".jsonl"}:
+            dataset = load_dataset("json", data_files=path, **(data_args or {}))
+        elif path.suffix.lower() == ".parquet":
+            dataset = load_dataset("parquet", data_files=path, **(data_args or {}))
+        elif path.suffix.lower() == ".arrow":
+            dataset = load_dataset("arrow", data_files=path, **(data_args or {}))
+        elif path.suffix.lower() == ".hdf5":
+            dataset = Dataset.from_pandas(pd.read_hdf(path), **(data_args or {}))
+        elif path.suffix.lower() == ".db":
+            dataset = Dataset.from_sql(con=path, **(data_args or {}))
+        elif path.suffix.lower() == ".tar":
+            dataset = load_dataset("webdataset", data_files=path, **(data_args or {}))
+        else:
+            raise ValueError(f"Unsupported file type: {path.suffix} given for {path}. ")
+
+        return dataset

guidellm/dataset/hf_datasets.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from datasets import (
+    Dataset,
+    DatasetDict,
+    IterableDataset,
+    IterableDatasetDict,
+    get_dataset_config_info,
+    load_dataset,
+)
+from transformers import PreTrainedTokenizerBase  # type: ignore[import]
+
+from guidellm.dataset.creator import DatasetCreator
+
+__all__ = ["HFDatasetsCreator"]
+
+
+class HFDatasetsCreator(DatasetCreator):
+    @classmethod
+    def is_supported(cls, data: Any, data_args: Optional[dict[str, Any]]) -> bool:  # noqa: ARG003
+        if isinstance(
+            data, (Dataset, DatasetDict, IterableDataset, IterableDatasetDict)
+        ):
+            # base type is supported
+            return True
+
+        if isinstance(data, (str, Path)) and (path := Path(data)).exists():
+            # local folder or py file, assume supported
+            return path.is_dir() or path.suffix == ".py"
+
+        if isinstance(data, (str, Path)):
+            try:
+                # try to load dataset
+                return get_dataset_config_info(data) is not None
+            except Exception:  # noqa: BLE001, S110
+                pass
+
+        return False
+
+    @classmethod
+    def handle_create(
+        cls,
+        data: Any,
+        data_args: Optional[dict[str, Any]],
+        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],  # noqa: ARG003
+        processor_args: Optional[dict[str, Any]],  # noqa: ARG003
+        random_seed: int,  # noqa: ARG003
+    ) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
+        if isinstance(data, (str, Path)):
+            data = load_dataset(data, **(data_args or {}))
+        elif data_args:
+            raise ValueError(
+                f"data_args should not be provided when data is a {type(data)}"
+            )
+
+        if isinstance(
+            data, (Dataset, DatasetDict, IterableDataset, IterableDatasetDict)
+        ):
+            return data
+
+        raise ValueError(f"Unsupported data type: {type(data)} given for {data}. ")