guidellm 0.4.0a21__py3-none-any.whl → 0.4.0a155__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of guidellm might be problematic. Click here for more details.
- guidellm/__init__.py +5 -2
- guidellm/__main__.py +451 -252
- guidellm/backends/__init__.py +33 -0
- guidellm/backends/backend.py +110 -0
- guidellm/backends/openai.py +355 -0
- guidellm/backends/response_handlers.py +455 -0
- guidellm/benchmark/__init__.py +53 -39
- guidellm/benchmark/benchmarker.py +148 -317
- guidellm/benchmark/entrypoints.py +466 -128
- guidellm/benchmark/output.py +517 -771
- guidellm/benchmark/profile.py +580 -280
- guidellm/benchmark/progress.py +568 -549
- guidellm/benchmark/scenarios/__init__.py +40 -0
- guidellm/benchmark/scenarios/chat.json +6 -0
- guidellm/benchmark/scenarios/rag.json +6 -0
- guidellm/benchmark/schemas.py +2085 -0
- guidellm/data/__init__.py +28 -4
- guidellm/data/collators.py +16 -0
- guidellm/data/deserializers/__init__.py +53 -0
- guidellm/data/deserializers/deserializer.py +109 -0
- guidellm/data/deserializers/file.py +222 -0
- guidellm/data/deserializers/huggingface.py +94 -0
- guidellm/data/deserializers/memory.py +192 -0
- guidellm/data/deserializers/synthetic.py +346 -0
- guidellm/data/loaders.py +145 -0
- guidellm/data/preprocessors/__init__.py +25 -0
- guidellm/data/preprocessors/formatters.py +412 -0
- guidellm/data/preprocessors/mappers.py +198 -0
- guidellm/data/preprocessors/preprocessor.py +29 -0
- guidellm/data/processor.py +30 -0
- guidellm/data/schemas.py +13 -0
- guidellm/data/utils/__init__.py +10 -0
- guidellm/data/utils/dataset.py +94 -0
- guidellm/data/utils/functions.py +18 -0
- guidellm/extras/__init__.py +4 -0
- guidellm/extras/audio.py +215 -0
- guidellm/extras/vision.py +242 -0
- guidellm/logger.py +2 -2
- guidellm/mock_server/__init__.py +8 -0
- guidellm/mock_server/config.py +84 -0
- guidellm/mock_server/handlers/__init__.py +17 -0
- guidellm/mock_server/handlers/chat_completions.py +280 -0
- guidellm/mock_server/handlers/completions.py +280 -0
- guidellm/mock_server/handlers/tokenizer.py +142 -0
- guidellm/mock_server/models.py +510 -0
- guidellm/mock_server/server.py +168 -0
- guidellm/mock_server/utils.py +302 -0
- guidellm/preprocess/dataset.py +23 -26
- guidellm/presentation/builder.py +2 -2
- guidellm/presentation/data_models.py +25 -21
- guidellm/presentation/injector.py +2 -3
- guidellm/scheduler/__init__.py +65 -26
- guidellm/scheduler/constraints.py +1035 -0
- guidellm/scheduler/environments.py +252 -0
- guidellm/scheduler/scheduler.py +140 -368
- guidellm/scheduler/schemas.py +272 -0
- guidellm/scheduler/strategies.py +519 -0
- guidellm/scheduler/worker.py +391 -420
- guidellm/scheduler/worker_group.py +707 -0
- guidellm/schemas/__init__.py +31 -0
- guidellm/schemas/info.py +159 -0
- guidellm/schemas/request.py +216 -0
- guidellm/schemas/response.py +119 -0
- guidellm/schemas/stats.py +228 -0
- guidellm/{config.py → settings.py} +32 -21
- guidellm/utils/__init__.py +95 -8
- guidellm/utils/auto_importer.py +98 -0
- guidellm/utils/cli.py +46 -2
- guidellm/utils/console.py +183 -0
- guidellm/utils/encoding.py +778 -0
- guidellm/utils/functions.py +134 -0
- guidellm/utils/hf_datasets.py +1 -2
- guidellm/utils/hf_transformers.py +4 -4
- guidellm/utils/imports.py +9 -0
- guidellm/utils/messaging.py +1118 -0
- guidellm/utils/mixins.py +115 -0
- guidellm/utils/pydantic_utils.py +411 -0
- guidellm/utils/random.py +3 -4
- guidellm/utils/registry.py +220 -0
- guidellm/utils/singleton.py +133 -0
- guidellm/{objects → utils}/statistics.py +341 -247
- guidellm/utils/synchronous.py +159 -0
- guidellm/utils/text.py +163 -50
- guidellm/utils/typing.py +41 -0
- guidellm/version.py +1 -1
- {guidellm-0.4.0a21.dist-info → guidellm-0.4.0a155.dist-info}/METADATA +33 -10
- guidellm-0.4.0a155.dist-info/RECORD +96 -0
- guidellm/backend/__init__.py +0 -23
- guidellm/backend/backend.py +0 -259
- guidellm/backend/openai.py +0 -705
- guidellm/backend/response.py +0 -136
- guidellm/benchmark/aggregator.py +0 -760
- guidellm/benchmark/benchmark.py +0 -837
- guidellm/benchmark/scenario.py +0 -104
- guidellm/data/prideandprejudice.txt.gz +0 -0
- guidellm/dataset/__init__.py +0 -22
- guidellm/dataset/creator.py +0 -213
- guidellm/dataset/entrypoints.py +0 -42
- guidellm/dataset/file.py +0 -92
- guidellm/dataset/hf_datasets.py +0 -62
- guidellm/dataset/in_memory.py +0 -132
- guidellm/dataset/synthetic.py +0 -287
- guidellm/objects/__init__.py +0 -18
- guidellm/objects/pydantic.py +0 -89
- guidellm/request/__init__.py +0 -18
- guidellm/request/loader.py +0 -284
- guidellm/request/request.py +0 -79
- guidellm/request/types.py +0 -10
- guidellm/scheduler/queues.py +0 -25
- guidellm/scheduler/result.py +0 -155
- guidellm/scheduler/strategy.py +0 -495
- guidellm-0.4.0a21.dist-info/RECORD +0 -62
- {guidellm-0.4.0a21.dist-info → guidellm-0.4.0a155.dist-info}/WHEEL +0 -0
- {guidellm-0.4.0a21.dist-info → guidellm-0.4.0a155.dist-info}/entry_points.txt +0 -0
- {guidellm-0.4.0a21.dist-info → guidellm-0.4.0a155.dist-info}/licenses/LICENSE +0 -0
- {guidellm-0.4.0a21.dist-info → guidellm-0.4.0a155.dist-info}/top_level.txt +0 -0
guidellm/scheduler/strategy.py
DELETED
|
@@ -1,495 +0,0 @@
|
|
|
1
|
-
import math
|
|
2
|
-
import random
|
|
3
|
-
import time
|
|
4
|
-
from collections.abc import Generator
|
|
5
|
-
from typing import (
|
|
6
|
-
Literal,
|
|
7
|
-
Optional,
|
|
8
|
-
Union,
|
|
9
|
-
)
|
|
10
|
-
|
|
11
|
-
from pydantic import Field
|
|
12
|
-
|
|
13
|
-
from guidellm.config import settings
|
|
14
|
-
from guidellm.objects import StandardBaseModel
|
|
15
|
-
|
|
16
|
-
__all__ = [
|
|
17
|
-
"AsyncConstantStrategy",
|
|
18
|
-
"AsyncPoissonStrategy",
|
|
19
|
-
"ConcurrentStrategy",
|
|
20
|
-
"SchedulingStrategy",
|
|
21
|
-
"StrategyType",
|
|
22
|
-
"SynchronousStrategy",
|
|
23
|
-
"ThroughputStrategy",
|
|
24
|
-
"strategy_display_str",
|
|
25
|
-
]
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
StrategyType = Literal["synchronous", "concurrent", "throughput", "constant", "poisson"]
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class SchedulingStrategy(StandardBaseModel):
|
|
32
|
-
"""
|
|
33
|
-
An abstract base class for scheduling strategies.
|
|
34
|
-
This class defines the interface for scheduling requests and provides
|
|
35
|
-
a common structure for all scheduling strategies.
|
|
36
|
-
Subclasses should implement the `request_times` method to provide
|
|
37
|
-
specific scheduling behavior.
|
|
38
|
-
|
|
39
|
-
:param type_: The type of scheduling strategy to use.
|
|
40
|
-
This should be one of the predefined strategy types.
|
|
41
|
-
"""
|
|
42
|
-
|
|
43
|
-
type_: Literal["strategy"] = Field(
|
|
44
|
-
description="The type of scheduling strategy schedule requests with.",
|
|
45
|
-
)
|
|
46
|
-
start_time: float = Field(
|
|
47
|
-
default_factory=time.time,
|
|
48
|
-
description="The start time for the scheduling strategy.",
|
|
49
|
-
)
|
|
50
|
-
|
|
51
|
-
@property
|
|
52
|
-
def processing_mode(self) -> Literal["sync", "async"]:
|
|
53
|
-
"""
|
|
54
|
-
The processing mode for the scheduling strategy, either 'sync' or 'async'.
|
|
55
|
-
This property determines how the worker processes are setup:
|
|
56
|
-
either to run synchronously with one request at a time or asynchronously.
|
|
57
|
-
This property should be implemented by subclasses to return
|
|
58
|
-
the appropriate processing mode.
|
|
59
|
-
|
|
60
|
-
:return: The processing mode for the scheduling strategy,
|
|
61
|
-
either 'sync' or 'async'.
|
|
62
|
-
"""
|
|
63
|
-
return "async"
|
|
64
|
-
|
|
65
|
-
@property
|
|
66
|
-
def processes_limit(self) -> int:
|
|
67
|
-
"""
|
|
68
|
-
The limit on the number of worker processes for the scheduling strategy.
|
|
69
|
-
It determines how many worker processes are created
|
|
70
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
71
|
-
|
|
72
|
-
:return: The number of processes for the scheduling strategy.
|
|
73
|
-
"""
|
|
74
|
-
return settings.max_worker_processes
|
|
75
|
-
|
|
76
|
-
@property
|
|
77
|
-
def queued_requests_limit(self) -> Optional[int]:
|
|
78
|
-
"""
|
|
79
|
-
The maximum number of queued requests for the scheduling strategy.
|
|
80
|
-
It determines how many requests can be queued at one time
|
|
81
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
82
|
-
|
|
83
|
-
:return: The maximum number of queued requests for the scheduling strategy.
|
|
84
|
-
"""
|
|
85
|
-
return settings.max_concurrency
|
|
86
|
-
|
|
87
|
-
@property
|
|
88
|
-
def processing_requests_limit(self) -> int:
|
|
89
|
-
"""
|
|
90
|
-
The maximum number of processing requests for the scheduling strategy.
|
|
91
|
-
It determines how many requests can be processed at one time
|
|
92
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
93
|
-
|
|
94
|
-
:return: The maximum number of processing requests for the scheduling strategy.
|
|
95
|
-
"""
|
|
96
|
-
return settings.max_concurrency
|
|
97
|
-
|
|
98
|
-
def request_times(self) -> Generator[float, None, None]:
|
|
99
|
-
"""
|
|
100
|
-
A generator that yields timestamps for when requests should be sent.
|
|
101
|
-
This method should be implemented by subclasses to provide specific
|
|
102
|
-
scheduling behavior.
|
|
103
|
-
|
|
104
|
-
:return: A generator that yields timestamps for request scheduling
|
|
105
|
-
or -1 for requests that should be sent immediately.
|
|
106
|
-
"""
|
|
107
|
-
raise NotImplementedError("Subclasses must implement request_times() method.")
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
class SynchronousStrategy(SchedulingStrategy):
|
|
111
|
-
"""
|
|
112
|
-
A class representing a synchronous scheduling strategy.
|
|
113
|
-
This strategy schedules requests synchronously, one at a time,
|
|
114
|
-
with the maximum rate possible.
|
|
115
|
-
It inherits from the `SchedulingStrategy` base class and
|
|
116
|
-
implements the `request_times` method to provide the specific
|
|
117
|
-
behavior for synchronous scheduling.
|
|
118
|
-
|
|
119
|
-
:param type_: The synchronous StrategyType to schedule requests synchronously.
|
|
120
|
-
"""
|
|
121
|
-
|
|
122
|
-
type_: Literal["synchronous"] = "synchronous" # type: ignore[assignment]
|
|
123
|
-
|
|
124
|
-
@property
|
|
125
|
-
def processing_mode(self) -> Literal["sync"]:
|
|
126
|
-
"""
|
|
127
|
-
The processing mode for the scheduling strategy, either 'sync' or 'async'.
|
|
128
|
-
This property determines how the worker processes are setup:
|
|
129
|
-
either to run synchronously with one request at a time or asynchronously.
|
|
130
|
-
|
|
131
|
-
:return: 'sync' for synchronous scheduling strategy
|
|
132
|
-
for the single worker process.
|
|
133
|
-
"""
|
|
134
|
-
return "sync"
|
|
135
|
-
|
|
136
|
-
@property
|
|
137
|
-
def processes_limit(self) -> int:
|
|
138
|
-
"""
|
|
139
|
-
The limit on the number of worker processes for the scheduling strategy.
|
|
140
|
-
It determines how many worker processes are created
|
|
141
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
142
|
-
|
|
143
|
-
:return: 1 for the synchronous scheduling strategy to limit
|
|
144
|
-
the worker processes to one.
|
|
145
|
-
"""
|
|
146
|
-
return 1
|
|
147
|
-
|
|
148
|
-
@property
|
|
149
|
-
def queued_requests_limit(self) -> int:
|
|
150
|
-
"""
|
|
151
|
-
The maximum number of queued requests for the scheduling strategy.
|
|
152
|
-
It determines how many requests can be queued at one time
|
|
153
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
154
|
-
|
|
155
|
-
:return: 1 for the synchronous scheduling strategy to limit
|
|
156
|
-
the queued requests to one that is ready to be processed.
|
|
157
|
-
"""
|
|
158
|
-
return 1
|
|
159
|
-
|
|
160
|
-
@property
|
|
161
|
-
def processing_requests_limit(self) -> int:
|
|
162
|
-
"""
|
|
163
|
-
The maximum number of processing requests for the scheduling strategy.
|
|
164
|
-
It determines how many requests can be processed at one time
|
|
165
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
166
|
-
|
|
167
|
-
:return: 1 for the synchronous scheduling strategy to limit
|
|
168
|
-
the processing requests to one that is ready to be processed.
|
|
169
|
-
"""
|
|
170
|
-
return 1
|
|
171
|
-
|
|
172
|
-
def request_times(self) -> Generator[float, None, None]:
|
|
173
|
-
"""
|
|
174
|
-
A generator that yields time.time() so requests are sent immediately,
|
|
175
|
-
while scheduling them synchronously.
|
|
176
|
-
|
|
177
|
-
:return: A generator that yields time.time() for immediate request scheduling.
|
|
178
|
-
"""
|
|
179
|
-
init_time = self.start_time
|
|
180
|
-
while True:
|
|
181
|
-
yield max(init_time, time.time())
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
class ConcurrentStrategy(SchedulingStrategy):
|
|
185
|
-
"""
|
|
186
|
-
A class representing a concurrent scheduling strategy.
|
|
187
|
-
This strategy schedules requests concurrently with the specified
|
|
188
|
-
number of streams.
|
|
189
|
-
It inherits from the `SchedulingStrategy` base class and
|
|
190
|
-
implements the `request_times` method to provide the specific
|
|
191
|
-
behavior for concurrent scheduling.
|
|
192
|
-
|
|
193
|
-
:param type_: The concurrent StrategyType to schedule requests concurrently.
|
|
194
|
-
:param streams: The number of concurrent streams to use for scheduling requests.
|
|
195
|
-
Each stream runs synchronously with the maximum rate possible.
|
|
196
|
-
This must be a positive integer.
|
|
197
|
-
"""
|
|
198
|
-
|
|
199
|
-
type_: Literal["concurrent"] = "concurrent" # type: ignore[assignment]
|
|
200
|
-
streams: int = Field(
|
|
201
|
-
description=(
|
|
202
|
-
"The number of concurrent streams to use for scheduling requests. "
|
|
203
|
-
"Each stream runs sychronously with the maximum rate possible. "
|
|
204
|
-
"This must be a positive integer."
|
|
205
|
-
),
|
|
206
|
-
gt=0,
|
|
207
|
-
)
|
|
208
|
-
|
|
209
|
-
@property
|
|
210
|
-
def processing_mode(self) -> Literal["sync"]:
|
|
211
|
-
"""
|
|
212
|
-
The processing mode for the scheduling strategy, either 'sync' or 'async'.
|
|
213
|
-
This property determines how the worker processes are setup:
|
|
214
|
-
either to run synchronously with one request at a time or asynchronously.
|
|
215
|
-
|
|
216
|
-
:return: 'sync' for synchronous scheduling strategy
|
|
217
|
-
for the multiple worker processes equal to streams.
|
|
218
|
-
"""
|
|
219
|
-
return "sync"
|
|
220
|
-
|
|
221
|
-
@property
|
|
222
|
-
def processes_limit(self) -> int:
|
|
223
|
-
"""
|
|
224
|
-
The limit on the number of worker processes for the scheduling strategy.
|
|
225
|
-
It determines how many worker processes are created
|
|
226
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
227
|
-
|
|
228
|
-
:return: {self.streams} for the concurrent scheduling strategy to limit
|
|
229
|
-
the worker processes to the number of streams.
|
|
230
|
-
"""
|
|
231
|
-
|
|
232
|
-
return min(self.streams, settings.max_worker_processes)
|
|
233
|
-
|
|
234
|
-
@property
|
|
235
|
-
def queued_requests_limit(self) -> int:
|
|
236
|
-
"""
|
|
237
|
-
The maximum number of queued requests for the scheduling strategy.
|
|
238
|
-
It determines how many requests can be queued at one time
|
|
239
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
240
|
-
|
|
241
|
-
:return: {self.streams} for the concurrent scheduling strategy to limit
|
|
242
|
-
the queued requests to the number of streams that are ready to be processed.
|
|
243
|
-
"""
|
|
244
|
-
return self.streams
|
|
245
|
-
|
|
246
|
-
@property
|
|
247
|
-
def processing_requests_limit(self) -> int:
|
|
248
|
-
"""
|
|
249
|
-
The maximum number of processing requests for the scheduling strategy.
|
|
250
|
-
It determines how many requests can be processed at one time
|
|
251
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
252
|
-
|
|
253
|
-
:return: {self.streams} for the concurrent scheduling strategy to limit
|
|
254
|
-
the processing requests to the number of streams that ready to be processed.
|
|
255
|
-
"""
|
|
256
|
-
return self.streams
|
|
257
|
-
|
|
258
|
-
def request_times(self) -> Generator[float, None, None]:
|
|
259
|
-
"""
|
|
260
|
-
A generator that yields time.time() so requests are sent
|
|
261
|
-
immediately, while scheduling them concurrently with the specified
|
|
262
|
-
number of streams.
|
|
263
|
-
|
|
264
|
-
:return: A generator that yields time.time() for immediate request scheduling.
|
|
265
|
-
"""
|
|
266
|
-
init_time = self.start_time
|
|
267
|
-
while True:
|
|
268
|
-
yield max(init_time, time.time())
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
class ThroughputStrategy(SchedulingStrategy):
|
|
272
|
-
"""
|
|
273
|
-
A class representing a throughput scheduling strategy.
|
|
274
|
-
This strategy schedules as many requests asynchronously as possible,
|
|
275
|
-
with the maximum rate possible.
|
|
276
|
-
It inherits from the `SchedulingStrategy` base class and
|
|
277
|
-
implements the `request_times` method to provide the specific
|
|
278
|
-
behavior for throughput scheduling.
|
|
279
|
-
|
|
280
|
-
:param type_: The throughput StrategyType to schedule requests asynchronously.
|
|
281
|
-
"""
|
|
282
|
-
|
|
283
|
-
type_: Literal["throughput"] = "throughput" # type: ignore[assignment]
|
|
284
|
-
max_concurrency: Optional[int] = Field(
|
|
285
|
-
default=None,
|
|
286
|
-
description=(
|
|
287
|
-
"The maximum number of concurrent requests to schedule. "
|
|
288
|
-
"If set to None, the concurrency value from settings will be used. "
|
|
289
|
-
"This must be a positive integer greater than 0."
|
|
290
|
-
),
|
|
291
|
-
gt=0,
|
|
292
|
-
)
|
|
293
|
-
|
|
294
|
-
@property
|
|
295
|
-
def processing_mode(self) -> Literal["async"]:
|
|
296
|
-
"""
|
|
297
|
-
The processing mode for the scheduling strategy, either 'sync' or 'async'.
|
|
298
|
-
This property determines how the worker processes are setup:
|
|
299
|
-
either to run synchronously with one request at a time or asynchronously.
|
|
300
|
-
|
|
301
|
-
:return: 'async' for asynchronous scheduling strategy
|
|
302
|
-
for the multiple worker processes handling requests.
|
|
303
|
-
"""
|
|
304
|
-
return "async"
|
|
305
|
-
|
|
306
|
-
@property
|
|
307
|
-
def queued_requests_limit(self) -> int:
|
|
308
|
-
"""
|
|
309
|
-
The maximum number of queued requests for the scheduling strategy.
|
|
310
|
-
It determines how many requests can be queued at one time
|
|
311
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
312
|
-
|
|
313
|
-
:return: The processing requests limit to ensure that there are enough
|
|
314
|
-
requests even for the worst case scenario where the max concurrent
|
|
315
|
-
requests are pulled at once for processing.
|
|
316
|
-
"""
|
|
317
|
-
return self.processing_requests_limit
|
|
318
|
-
|
|
319
|
-
@property
|
|
320
|
-
def processing_requests_limit(self) -> int:
|
|
321
|
-
"""
|
|
322
|
-
The maximum number of processing requests for the scheduling strategy.
|
|
323
|
-
It determines how many requests can be processed at one time
|
|
324
|
-
for the scheduling strategy and must be implemented by subclasses.
|
|
325
|
-
|
|
326
|
-
:return: {self.max_concurrency} for the throughput scheduling strategy to limit
|
|
327
|
-
the processing requests to the maximum concurrency.
|
|
328
|
-
If max_concurrency is None, then the default processing requests limit
|
|
329
|
-
will be used.
|
|
330
|
-
"""
|
|
331
|
-
return self.max_concurrency or super().processing_requests_limit
|
|
332
|
-
|
|
333
|
-
def request_times(self) -> Generator[float, None, None]:
|
|
334
|
-
"""
|
|
335
|
-
A generator that yields the start time.time() so requests are sent
|
|
336
|
-
immediately, while scheduling as many asynchronously as possible.
|
|
337
|
-
|
|
338
|
-
:return: A generator that yields the start time.time()
|
|
339
|
-
for immediate request scheduling.
|
|
340
|
-
"""
|
|
341
|
-
init_time = self.start_time
|
|
342
|
-
while True:
|
|
343
|
-
yield init_time
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
class AsyncConstantStrategy(ThroughputStrategy):
|
|
347
|
-
"""
|
|
348
|
-
A class representing an asynchronous constant scheduling strategy.
|
|
349
|
-
This strategy schedules requests asynchronously at a constant request rate
|
|
350
|
-
in requests per second.
|
|
351
|
-
If initial_burst is set, it will send an initial burst of math.floor(rate)
|
|
352
|
-
requests to reach the target rate.
|
|
353
|
-
This is useful to ensure that the target rate is reached quickly
|
|
354
|
-
and then maintained.
|
|
355
|
-
It inherits from the `SchedulingStrategy` base class and
|
|
356
|
-
implements the `request_times` method to provide the specific
|
|
357
|
-
behavior for asynchronous constant scheduling.
|
|
358
|
-
|
|
359
|
-
:param type_: The constant StrategyType to schedule requests asynchronously.
|
|
360
|
-
:param rate: The rate at which to schedule requests asynchronously in
|
|
361
|
-
requests per second. This must be a positive float.
|
|
362
|
-
:param initial_burst: True to send an initial burst of requests
|
|
363
|
-
(math.floor(self.rate)) to reach target rate.
|
|
364
|
-
False to not send an initial burst.
|
|
365
|
-
"""
|
|
366
|
-
|
|
367
|
-
type_: Literal["constant"] = "constant" # type: ignore[assignment]
|
|
368
|
-
rate: float = Field(
|
|
369
|
-
description=(
|
|
370
|
-
"The rate at which to schedule requests asynchronously in "
|
|
371
|
-
"requests per second. This must be a positive float."
|
|
372
|
-
),
|
|
373
|
-
gt=0,
|
|
374
|
-
)
|
|
375
|
-
initial_burst: bool = Field(
|
|
376
|
-
default=True,
|
|
377
|
-
description=(
|
|
378
|
-
"True to send an initial burst of requests (math.floor(self.rate)) "
|
|
379
|
-
"to reach target rate. False to not send an initial burst."
|
|
380
|
-
),
|
|
381
|
-
)
|
|
382
|
-
|
|
383
|
-
def request_times(self) -> Generator[float, None, None]:
|
|
384
|
-
"""
|
|
385
|
-
A generator that yields timestamps for when requests should be sent.
|
|
386
|
-
This method schedules requests asynchronously at a constant rate
|
|
387
|
-
in requests per second.
|
|
388
|
-
If burst_time is set, it will send an initial burst of requests
|
|
389
|
-
to reach the target rate.
|
|
390
|
-
This is useful to ensure that the target rate is reached quickly
|
|
391
|
-
and then maintained.
|
|
392
|
-
|
|
393
|
-
:return: A generator that yields timestamps for request scheduling.
|
|
394
|
-
"""
|
|
395
|
-
constant_increment = 1.0 / self.rate
|
|
396
|
-
|
|
397
|
-
init_time = self.start_time
|
|
398
|
-
# handle bursts first to get to the desired rate
|
|
399
|
-
if self.initial_burst is not None:
|
|
400
|
-
# send an initial burst equal to the rate
|
|
401
|
-
# to reach the target rate
|
|
402
|
-
burst_count = math.floor(self.rate)
|
|
403
|
-
for _ in range(burst_count):
|
|
404
|
-
yield init_time
|
|
405
|
-
|
|
406
|
-
init_time += constant_increment
|
|
407
|
-
|
|
408
|
-
counter = 0
|
|
409
|
-
|
|
410
|
-
# continue with constant rate after bursting
|
|
411
|
-
while True:
|
|
412
|
-
yield init_time + constant_increment * counter
|
|
413
|
-
counter += 1
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
class AsyncPoissonStrategy(ThroughputStrategy):
|
|
417
|
-
"""
|
|
418
|
-
A class representing an asynchronous Poisson scheduling strategy.
|
|
419
|
-
This strategy schedules requests asynchronously at a Poisson request rate
|
|
420
|
-
in requests per second.
|
|
421
|
-
If initial_burst is set, it will send an initial burst of math.floor(rate)
|
|
422
|
-
requests to reach the target rate.
|
|
423
|
-
It inherits from the `SchedulingStrategy` base class and
|
|
424
|
-
implements the `request_times` method to provide the specific
|
|
425
|
-
behavior for asynchronous Poisson scheduling.
|
|
426
|
-
|
|
427
|
-
:param type_: The Poisson StrategyType to schedule requests asynchronously.
|
|
428
|
-
:param rate: The rate at which to schedule requests asynchronously in
|
|
429
|
-
requests per second. This must be a positive float.
|
|
430
|
-
:param initial_burst: True to send an initial burst of requests
|
|
431
|
-
(math.floor(self.rate)) to reach target rate.
|
|
432
|
-
False to not send an initial burst.
|
|
433
|
-
"""
|
|
434
|
-
|
|
435
|
-
type_: Literal["poisson"] = "poisson" # type: ignore[assignment]
|
|
436
|
-
rate: float = Field(
|
|
437
|
-
description=(
|
|
438
|
-
"The rate at which to schedule requests asynchronously in "
|
|
439
|
-
"requests per second. This must be a positive float."
|
|
440
|
-
),
|
|
441
|
-
gt=0,
|
|
442
|
-
)
|
|
443
|
-
initial_burst: bool = Field(
|
|
444
|
-
default=True,
|
|
445
|
-
description=(
|
|
446
|
-
"True to send an initial burst of requests (math.floor(self.rate)) "
|
|
447
|
-
"to reach target rate. False to not send an initial burst."
|
|
448
|
-
),
|
|
449
|
-
)
|
|
450
|
-
random_seed: int = Field(
|
|
451
|
-
default=42,
|
|
452
|
-
description=("The random seed to use for the Poisson distribution. "),
|
|
453
|
-
)
|
|
454
|
-
|
|
455
|
-
def request_times(self) -> Generator[float, None, None]:
|
|
456
|
-
"""
|
|
457
|
-
A generator that yields timestamps for when requests should be sent.
|
|
458
|
-
This method schedules requests asynchronously at a Poisson rate
|
|
459
|
-
in requests per second.
|
|
460
|
-
The inter arrival time between requests is exponentially distributed
|
|
461
|
-
based on the rate.
|
|
462
|
-
|
|
463
|
-
:return: A generator that yields timestamps for request scheduling.
|
|
464
|
-
"""
|
|
465
|
-
init_time = self.start_time
|
|
466
|
-
if self.initial_burst is not None:
|
|
467
|
-
# send an initial burst equal to the rate
|
|
468
|
-
# to reach the target rate
|
|
469
|
-
burst_count = math.floor(self.rate)
|
|
470
|
-
for _ in range(burst_count):
|
|
471
|
-
yield init_time
|
|
472
|
-
else:
|
|
473
|
-
yield init_time
|
|
474
|
-
|
|
475
|
-
# set the random seed for reproducibility
|
|
476
|
-
rand = random.Random(self.random_seed) # noqa: S311
|
|
477
|
-
|
|
478
|
-
while True:
|
|
479
|
-
inter_arrival_time = rand.expovariate(self.rate)
|
|
480
|
-
init_time += inter_arrival_time
|
|
481
|
-
yield init_time
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
def strategy_display_str(strategy: Union[StrategyType, SchedulingStrategy]) -> str:
|
|
485
|
-
strategy_type = strategy if isinstance(strategy, str) else strategy.type_
|
|
486
|
-
strategy_instance = strategy if isinstance(strategy, SchedulingStrategy) else None
|
|
487
|
-
|
|
488
|
-
if strategy_type == "concurrent":
|
|
489
|
-
rate = f"@{strategy_instance.streams}" if strategy_instance else "@##" # type: ignore[attr-defined]
|
|
490
|
-
elif strategy_type in ("constant", "poisson"):
|
|
491
|
-
rate = f"@{strategy_instance.rate:.2f}" if strategy_instance else "@#.##" # type: ignore[attr-defined]
|
|
492
|
-
else:
|
|
493
|
-
rate = ""
|
|
494
|
-
|
|
495
|
-
return f"{strategy_type}{rate}"
|
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
guidellm/__init__.py,sha256=qXCx-HonNByJ2PDKqOUnD7CcbxA7cazNKyqKigwyuyE,1139
|
|
2
|
-
guidellm/__main__.py,sha256=R068ZOcVQxzSsvq93s4ZZTsW6Qp0eEX7nlvwqtJSpYQ,15225
|
|
3
|
-
guidellm/config.py,sha256=C4F9LapOr4Xu8P-nXltURrPVAMqpPl-NZb3On4mzaBg,6671
|
|
4
|
-
guidellm/logger.py,sha256=mRLbC3jT8MnAN7_ZO00BAJKI33MX9QZgKUYaZ8NKOsQ,2909
|
|
5
|
-
guidellm/version.py,sha256=XZeUwR24DzG1AjuKV0s8cRyc0Xv8cPiqXYSZTt7xQVg,127
|
|
6
|
-
guidellm/backend/__init__.py,sha256=nQmxVNG9cDdNbY0NwjXxIRV4oyZKIhb6cGyuXOakFnk,489
|
|
7
|
-
guidellm/backend/backend.py,sha256=PGDDppp7JlcFl4gQWsvvfC3iZY376UFcw3Y9lKM-ydQ,9798
|
|
8
|
-
guidellm/backend/openai.py,sha256=SS0jjczGXUeuP4bUDzR5dZvg72Lv0rUY_KjBIT93cCk,27033
|
|
9
|
-
guidellm/backend/response.py,sha256=ipv5amHOGHK0fW4rOIt1d8Vj1yXTizSh3Ux5MaN96KY,5139
|
|
10
|
-
guidellm/benchmark/__init__.py,sha256=px1ngc7UCeTbCKAKij39576WMyz-5hri_Xvh7kQgs8I,1837
|
|
11
|
-
guidellm/benchmark/aggregator.py,sha256=E6BZpBIFv5AtWhv2u5XH-tLohuUYNI9ZcMobhCxlOJg,31312
|
|
12
|
-
guidellm/benchmark/benchmark.py,sha256=I4_VQI2vqVge94ZZL8lrEuNliHtB42g7Awf-ZVnudVM,31806
|
|
13
|
-
guidellm/benchmark/benchmarker.py,sha256=pwhndRmbuZtsglezGB_F5fcjGOQU9A9cw8lY8upRl_M,11738
|
|
14
|
-
guidellm/benchmark/entrypoints.py,sha256=eauxjJzAgM7oxcHX8lYcVIq9AkhYTkjK8CKpUQFvEo0,5817
|
|
15
|
-
guidellm/benchmark/output.py,sha256=UicODIA52eIfHbkveMwKLjf_J9qcpAnROTGSTMIha4s,36346
|
|
16
|
-
guidellm/benchmark/profile.py,sha256=A-vovIl39d1GiI4cOYgWai_99yj2ExWVlx88LAbIm5I,13091
|
|
17
|
-
guidellm/benchmark/progress.py,sha256=yydigV1E-jrF5cN__8m-1NiOlrdRaV68FgMBTZ7lK8E,25279
|
|
18
|
-
guidellm/benchmark/scenario.py,sha256=5DhqrcuIwwumLBD065yRvHQ-VBi2Vk3DWXPU1zeFGDE,3309
|
|
19
|
-
guidellm/benchmark/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
-
guidellm/data/__init__.py,sha256=sTZlJdpYcfFKkNk-CHtHm7_l8n9KwBhlM7qs247ELCY,120
|
|
21
|
-
guidellm/data/prideandprejudice.txt.gz,sha256=dmze5vnnSZmTU5NP7E1zcH30bOOnpYu5Motlm57xpYk,241795
|
|
22
|
-
guidellm/dataset/__init__.py,sha256=iWj58dDYdljYKHMeEVRol65YmsxC8C6ysPQw6v2mozw,593
|
|
23
|
-
guidellm/dataset/creator.py,sha256=bW70HNSybqWaCsNoeblwih0Y3mKgedmK_7ewbc14cxo,6614
|
|
24
|
-
guidellm/dataset/entrypoints.py,sha256=-mYCyhibHPLBXQ_FmZOY3Lg4tQhDNEOfT8U9rSQNDQI,1370
|
|
25
|
-
guidellm/dataset/file.py,sha256=hkRndn4WCaIrmWhje-Z81--JnPZVBhs7876P0Mo00mE,3350
|
|
26
|
-
guidellm/dataset/hf_datasets.py,sha256=oWHeC0Iup4E4QulWcZWiCcQ4jtjw9_sTINJ6EEBTCXo,2016
|
|
27
|
-
guidellm/dataset/in_memory.py,sha256=_p8CCW9xTTWG9aqbIE4mf9s3BVAZutneIPn-rEXT7xo,4699
|
|
28
|
-
guidellm/dataset/synthetic.py,sha256=os1STDSAlrMxnmR5w4Y3iOEE2j4DVYFOpIEWqpabk4g,9396
|
|
29
|
-
guidellm/objects/__init__.py,sha256=FJg5REEBg9hSuMAYnGAud9IxBcASpZuDwYDYEmwdnTY,384
|
|
30
|
-
guidellm/objects/pydantic.py,sha256=l19IHTJVIet0547xZ18jdcT44rI5Bmf19ASV3vpxww4,2889
|
|
31
|
-
guidellm/objects/statistics.py,sha256=0hiyvl3QFxlWzXb0xE9EN-fPxs6kPJiFEXKEyn4d3WM,37264
|
|
32
|
-
guidellm/preprocess/__init__.py,sha256=6mRs1atYwYkdX4txez_bEVk-_nCDsNt5Wo20eWZ24jA,112
|
|
33
|
-
guidellm/preprocess/dataset.py,sha256=P-TJEebcWnUFGczImc7AOi90vJ4sakZtU8NSYWhJXlM,12318
|
|
34
|
-
guidellm/presentation/__init__.py,sha256=tkkHf8ZB_rH-rqnAGY21NuOw2GQq3WCGix_whMqeUSs,483
|
|
35
|
-
guidellm/presentation/builder.py,sha256=kObRHGOp9D9D5GEVTIENxVF5S31JJ4MyCIQl_ue0xKc,894
|
|
36
|
-
guidellm/presentation/data_models.py,sha256=vXjB-lG-a7GBFevroapGpMezK1ytQHSBKy6Y6COlpWQ,7230
|
|
37
|
-
guidellm/presentation/injector.py,sha256=fiJQ3K9qcA7Lhje1rFFMz5MrMsn3O69pCW0N2n66yUU,1863
|
|
38
|
-
guidellm/request/__init__.py,sha256=afqgezgqrrSvGA95y-1rAMyETqgKm9l-a0hAAP0lepk,419
|
|
39
|
-
guidellm/request/loader.py,sha256=33-pYEXDwMJgaPuPWMUzJhIe0UpnQ7n5Zz84Qg637PA,9094
|
|
40
|
-
guidellm/request/request.py,sha256=BCf0Ua0h1oOj0y6HCyUh71r5tfFeS0fDrpaoU-UnMj4,3576
|
|
41
|
-
guidellm/request/types.py,sha256=SU4RnCclVqAg2lA1AOSFWf-_5PNcXwfqQUUwU8Wv_5I,141
|
|
42
|
-
guidellm/scheduler/__init__.py,sha256=e-haFdeLVEzIbkBeNafS-ZW-L4kUbLA9zswKvJDsYCc,1073
|
|
43
|
-
guidellm/scheduler/queues.py,sha256=gHDwMgz7g9hW_o1AP8fg2smXt69TOmux99K_pvdfnMI,621
|
|
44
|
-
guidellm/scheduler/result.py,sha256=4fxR0UGm7SByJV5Ptaf6BFlLZkMdtSlv1x8PgdVCHko,5603
|
|
45
|
-
guidellm/scheduler/scheduler.py,sha256=iwZNHzRvqbUggmjweB72Hs6pamFpO9_ykcxpCA7FwBk,14587
|
|
46
|
-
guidellm/scheduler/strategy.py,sha256=5NZG2E_KCe-YJD_EJ5ZJCYLh7luRhIg-s5l9beRax9w,18829
|
|
47
|
-
guidellm/scheduler/worker.py,sha256=x_hyqykIfkLSG-8S5xgNRI3qO57sADZhdakPojNVCsg,16345
|
|
48
|
-
guidellm/utils/__init__.py,sha256=cFC_HhEQdoYbJuVap_ORRY2rdlFRvdwgQwbB0QVI9nk,830
|
|
49
|
-
guidellm/utils/cli.py,sha256=t76C0q7-OFewzhFYyRZe-qR8b90FlcRm7wk3EjKiJZg,1961
|
|
50
|
-
guidellm/utils/colors.py,sha256=D0IGz8A346-Pt5qgnP3S5uV-VgngJoXbfToVCOna41k,175
|
|
51
|
-
guidellm/utils/default_group.py,sha256=iZ47bwRcUCxkX04Zdg0qpmqKtFg4P7lt5_hpw1CnKkA,4167
|
|
52
|
-
guidellm/utils/dict.py,sha256=oogh34_NznFEn1L6NKY2RDVBm7TUK9LOZfMc-rquNw8,673
|
|
53
|
-
guidellm/utils/hf_datasets.py,sha256=C99cB4StbhjC8XtnzLLGe6A0TYrs63EapQZJQmQr8dI,1023
|
|
54
|
-
guidellm/utils/hf_transformers.py,sha256=3iF40l02VEWOcS8kasO8TSws0Lp3cE-NyiqoB9GnHuA,1021
|
|
55
|
-
guidellm/utils/random.py,sha256=elA8HZ3AIN5T2pa7cgq35OVK__0SQmZVS4IzxJaOpvw,1310
|
|
56
|
-
guidellm/utils/text.py,sha256=yCcaJHQGS6b2eNAgx9L9nlJi9pXLhpxuqkLRXdsk04A,6545
|
|
57
|
-
guidellm-0.4.0a21.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
58
|
-
guidellm-0.4.0a21.dist-info/METADATA,sha256=D23usOvF8ChhE0sQpyxHIY0td88O8vOSJai5ArgGmTs,21096
|
|
59
|
-
guidellm-0.4.0a21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
60
|
-
guidellm-0.4.0a21.dist-info/entry_points.txt,sha256=DzLFEg47fF7qY1b-9laPz9jg0KSKJ1_D9TbF93kLz_E,51
|
|
61
|
-
guidellm-0.4.0a21.dist-info/top_level.txt,sha256=EXRGjnvFtL6MeZTe0tnHRMYcEWUW3vEqoG2zO7vFOtk,9
|
|
62
|
-
guidellm-0.4.0a21.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|