most-queue 2.9__py3-none-any.whl → 2.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- most_queue/io/serving.py +258 -0
- most_queue/random/feature_service.py +175 -0
- most_queue/random/marked_arrivals.py +127 -0
- most_queue/random/queue_selection.py +47 -0
- most_queue/random/service_calibration.py +91 -0
- most_queue/random/trace_resampling.py +145 -0
- most_queue/random/utils/fit.py +20 -7
- most_queue/sim/accumulating_priority.py +118 -0
- most_queue/sim/admission_control.py +90 -0
- most_queue/sim/base.py +1 -0
- most_queue/sim/base_core.py +33 -0
- most_queue/sim/bulk_map_finite.py +191 -0
- most_queue/sim/decaying_value.py +188 -0
- most_queue/sim/delay_dependent.py +115 -0
- most_queue/sim/dynamic_batching.py +236 -0
- most_queue/sim/edf.py +162 -0
- most_queue/sim/heterogeneous_bulk_service.py +237 -0
- most_queue/sim/imprecise.py +258 -0
- most_queue/sim/inventory.py +687 -0
- most_queue/sim/kv_cache_congestion.py +226 -0
- most_queue/sim/kv_cache_stability.py +219 -0
- most_queue/sim/load_balancing.py +98 -0
- most_queue/sim/msj_checkpoint.py +273 -0
- most_queue/sim/msj_general.py +535 -0
- most_queue/sim/msj_lifecycle.py +355 -0
- most_queue/sim/negative.py +13 -16
- most_queue/sim/networks/closed_network.py +147 -0
- most_queue/sim/networks/fork_join_network.py +167 -0
- most_queue/sim/networks/network.py +12 -2
- most_queue/sim/networks/tandem_blocking.py +142 -0
- most_queue/sim/networks/time_varying_network.py +124 -0
- most_queue/sim/polling.py +87 -0
- most_queue/sim/prefill_decode.py +234 -0
- most_queue/sim/priority.py +35 -0
- most_queue/sim/priority_heterogeneous.py +89 -0
- most_queue/sim/priority_impatience.py +105 -0
- most_queue/sim/reliability.py +413 -0
- most_queue/sim/retrial_priority.py +104 -0
- most_queue/sim/smdp_batching.py +170 -0
- most_queue/sim/time_varying.py +69 -0
- most_queue/sim/token_batching.py +181 -0
- most_queue/sim/utils/acme_trace.py +204 -0
- most_queue/sim/utils/llm_trace.py +101 -0
- most_queue/sim/utils/msj_calendar.py +66 -0
- most_queue/sim/utils/msj_capacity_calendar.py +104 -0
- most_queue/sim/utils/msj_packing.py +98 -0
- most_queue/sim/utils/residual_runtime.py +145 -0
- most_queue/sim/utils/resource_envelope.py +135 -0
- most_queue/sim/utils/runtime_prediction.py +239 -0
- most_queue/sim/utils/workload_lifecycle.py +101 -0
- most_queue/sim/utils/workload_trace.py +172 -0
- most_queue/sim/value_scheduling.py +439 -0
- most_queue/structs.py +456 -1
- most_queue/theory/__init__.py +2 -0
- most_queue/theory/admission_control/__init__.py +14 -0
- most_queue/theory/admission_control/mm1_deadline_admission.py +204 -0
- most_queue/theory/batch/__init__.py +3 -1
- most_queue/theory/batch/_idle_refill.py +240 -0
- most_queue/theory/batch/_tagged_batch.py +203 -0
- most_queue/theory/batch/_tagged_boundary.py +76 -0
- most_queue/theory/batch/bulk_service.py +261 -18
- most_queue/theory/batch/bulk_service_erlang.py +401 -0
- most_queue/theory/batch/bulk_service_general.py +79 -0
- most_queue/theory/batch/bulk_service_h2.py +546 -0
- most_queue/theory/batch/bulk_service_multiserver.py +697 -0
- most_queue/theory/batch/callback_batching.py +125 -0
- most_queue/theory/batch/dynamic_batching.py +396 -0
- most_queue/theory/batch/heterogeneous_bulk_service.py +369 -0
- most_queue/theory/batch/map_ph_finite_buffer.py +695 -0
- most_queue/theory/batch/smdp_batching.py +252 -0
- most_queue/theory/batch/token_batching.py +321 -0
- most_queue/theory/batch/token_service.py +276 -0
- most_queue/theory/continuous_batching/__init__.py +42 -0
- most_queue/theory/continuous_batching/core_aware_ttft.py +218 -0
- most_queue/theory/continuous_batching/infinite_stage_aware_ttft.py +255 -0
- most_queue/theory/continuous_batching/kv_cache_congestion.py +241 -0
- most_queue/theory/continuous_batching/kv_cache_stability.py +151 -0
- most_queue/theory/continuous_batching/occupancy_dependent.py +183 -0
- most_queue/theory/continuous_batching/occupancy_dependent_h2.py +272 -0
- most_queue/theory/continuous_batching/phase_aware_stability.py +261 -0
- most_queue/theory/continuous_batching/phase_aware_ttft.py +157 -0
- most_queue/theory/continuous_batching/prefill_decode.py +211 -0
- most_queue/theory/continuous_batching/reduced_stage_aware_ttft.py +182 -0
- most_queue/theory/continuous_batching/serving_calibration.py +147 -0
- most_queue/theory/continuous_batching/stage_aware_ttft.py +245 -0
- most_queue/theory/continuous_batching/transient_occupancy.py +204 -0
- most_queue/theory/decaying_value/__init__.py +22 -0
- most_queue/theory/decaying_value/service_rate_control.py +419 -0
- most_queue/theory/delay_dependent/__init__.py +13 -0
- most_queue/theory/delay_dependent/mmc_threshold_service.py +494 -0
- most_queue/theory/fifo/__init__.py +2 -0
- most_queue/theory/fifo/_ps_sojourn.py +244 -0
- most_queue/theory/fifo/gi_m2_state_dependent.py +307 -0
- most_queue/theory/fifo/mg1_ps.py +188 -7
- most_queue/theory/fifo/mgn_ps.py +140 -0
- most_queue/theory/fifo/mgn_takahasi.py +1 -1
- most_queue/theory/fifo/mm1_threshold_rate.py +299 -0
- most_queue/theory/fork_join/__init__.py +2 -0
- most_queue/theory/fork_join/dag.py +135 -0
- most_queue/theory/fork_join/split_join.py +46 -9
- most_queue/theory/imprecise/__init__.py +26 -0
- most_queue/theory/imprecise/offline.py +345 -0
- most_queue/theory/inventory/__init__.py +26 -0
- most_queue/theory/inventory/_wait_phase.py +431 -0
- most_queue/theory/inventory/mm1_inventory.py +285 -0
- most_queue/theory/inventory/mm1_inventory_erlang_replenishment.py +272 -0
- most_queue/theory/inventory/mm2_heterogeneous_inventory.py +294 -0
- most_queue/theory/inventory/mmc_heterogeneous_erlang_inventory.py +399 -0
- most_queue/theory/inventory/mmc_heterogeneous_h2_inventory.py +394 -0
- most_queue/theory/inventory/mmc_heterogeneous_inventory.py +348 -0
- most_queue/theory/inventory/mmc_inventory.py +351 -0
- most_queue/theory/load_balancing.py +99 -0
- most_queue/theory/msj/__init__.py +2 -1
- most_queue/theory/msj/ph.py +285 -0
- most_queue/theory/negative/mgn_disaster.py +37 -58
- most_queue/theory/negative/mgn_rcs.py +36 -202
- most_queue/theory/networks/__init__.py +22 -1
- most_queue/theory/networks/bcmp_network.py +283 -0
- most_queue/theory/networks/blocking.py +153 -0
- most_queue/theory/networks/closed_network.py +270 -0
- most_queue/theory/networks/fork_join_network.py +115 -0
- most_queue/theory/networks/g_network.py +259 -0
- most_queue/theory/networks/jackson_network.py +116 -0
- most_queue/theory/networks/qna.py +185 -0
- most_queue/theory/networks/time_varying_network.py +94 -0
- most_queue/theory/networks/traffic.py +23 -0
- most_queue/theory/polling.py +114 -0
- most_queue/theory/priority/__init__.py +24 -3
- most_queue/theory/priority/accumulating.py +108 -0
- most_queue/theory/priority/impatience.py +140 -0
- most_queue/theory/priority/map_ph_priority.py +235 -0
- most_queue/theory/priority/preemptive/mg1_repeat.py +155 -0
- most_queue/theory/priority/preemptive/mm2_heterogeneous.py +240 -0
- most_queue/theory/priority/retrial_priority.py +132 -0
- most_queue/theory/reliability/__init__.py +21 -0
- most_queue/theory/reliability/machine_repair.py +104 -0
- most_queue/theory/reliability/machine_repair_heterogeneous.py +150 -0
- most_queue/theory/reliability/mm1_disaster_repair.py +102 -0
- most_queue/theory/reliability/mm1_working_breakdowns.py +108 -0
- most_queue/theory/reliability/mmc_breakdowns.py +119 -0
- most_queue/theory/reliability/retrial_unreliable.py +110 -0
- most_queue/theory/reliability/utils.py +47 -0
- most_queue/theory/srpt/utils/load_below.py +13 -34
- most_queue/theory/time_varying.py +123 -0
- most_queue/theory/utils/__init__.py +12 -0
- most_queue/theory/utils/max_dist.py +236 -0
- most_queue/theory/utils/restarts.py +90 -14
- most_queue/theory/utils/sla.py +146 -0
- most_queue/theory/utils/tail.py +46 -0
- most_queue/theory/value_scheduling/__init__.py +25 -0
- most_queue/theory/value_scheduling/offline.py +336 -0
- most_queue-2.11.dist-info/METADATA +635 -0
- most_queue-2.11.dist-info/RECORD +301 -0
- {most_queue-2.9.dist-info → most_queue-2.11.dist-info}/WHEEL +1 -1
- most_queue-2.9.dist-info/METADATA +0 -200
- most_queue-2.9.dist-info/RECORD +0 -174
- {most_queue-2.9.dist-info → most_queue-2.11.dist-info}/licenses/LICENSE +0 -0
most_queue/io/serving.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Audited request-level serving measurements and joint TTFT/TPOT SLOs.
|
|
2
|
+
|
|
3
|
+
Reference serializer: vLLM v0.33.0rc3, commit
|
|
4
|
+
4ba787ec9959e566d3b587a796dff07673324da5, vllm/benchmarks/serve.py.
|
|
5
|
+
TTFT and latency are client measurements; queue_times is CLIENT semaphore
|
|
6
|
+
waiting. Server timings are separate optional measurements, never inferred.
|
|
7
|
+
The SGLang array reader explicitly uses the span of streamed output events
|
|
8
|
+
when full per-request latencies are absent. See docs/serving_measurements.md.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ServingRequest:
|
|
21
|
+
"""One offered request; all durations are seconds and counts are actual.
|
|
22
|
+
|
|
23
|
+
A pending request has an unknown final latency/TPOT. Failed and cancelled
|
|
24
|
+
requests remain in the offered denominator. An absent measurement is None.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
input_tokens: int
|
|
28
|
+
output_tokens: int
|
|
29
|
+
status: str
|
|
30
|
+
ttft: float | None = None
|
|
31
|
+
latency: float | None = None
|
|
32
|
+
client_queue_time: float | None = None
|
|
33
|
+
server_queue_time: float | None = None
|
|
34
|
+
server_ttft: float | None = None
|
|
35
|
+
|
|
36
|
+
def __post_init__(self):
|
|
37
|
+
if self.status not in {"completed", "failed", "cancelled", "pending"}:
|
|
38
|
+
raise ValueError("unknown request status")
|
|
39
|
+
for count in (self.input_tokens, self.output_tokens):
|
|
40
|
+
if isinstance(count, bool) or not isinstance(count, (int, np.integer)) or count < 0:
|
|
41
|
+
raise ValueError("token counts must be nonnegative integers")
|
|
42
|
+
for value in (self.ttft, self.latency, self.client_queue_time, self.server_queue_time, self.server_ttft):
|
|
43
|
+
if value is not None and (not math.isfinite(value) or value < 0):
|
|
44
|
+
raise ValueError("measured durations must be finite and nonnegative, or None")
|
|
45
|
+
if self.status == "completed":
|
|
46
|
+
if min(self.input_tokens, self.output_tokens) < 1 or self.ttft is None or self.latency is None:
|
|
47
|
+
raise ValueError("completed text requests require positive lengths, TTFT and latency")
|
|
48
|
+
if self.latency < self.ttft:
|
|
49
|
+
raise ValueError("latency cannot precede the first token")
|
|
50
|
+
elif self.latency is not None:
|
|
51
|
+
raise ValueError("only completed requests have a final latency")
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def tpot(self):
|
|
55
|
+
"""Per-request TPOT, excluding the first token; None for <=1 token."""
|
|
56
|
+
if self.status != "completed" or self.output_tokens <= 1:
|
|
57
|
+
return None
|
|
58
|
+
return (self.latency - self.ttft) / (self.output_tokens - 1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class ServingBenchmark:
|
|
63
|
+
"""A finite offered cohort and its explicit measurement-window duration."""
|
|
64
|
+
|
|
65
|
+
requests: tuple[ServingRequest, ...]
|
|
66
|
+
duration: float
|
|
67
|
+
metadata: dict = field(default_factory=dict)
|
|
68
|
+
|
|
69
|
+
def __post_init__(self):
|
|
70
|
+
if not self.requests or not all(isinstance(row, ServingRequest) for row in self.requests):
|
|
71
|
+
raise ValueError("a benchmark requires a nonempty request cohort")
|
|
72
|
+
if not math.isfinite(self.duration) or self.duration <= 0:
|
|
73
|
+
raise ValueError("duration must be finite and positive")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _read_arrays(source, schema):
|
|
77
|
+
data = json.loads(Path(source).read_text(encoding="utf-8")) if isinstance(source, (str, Path)) else source
|
|
78
|
+
if not isinstance(data, dict):
|
|
79
|
+
raise ValueError("expected a single benchmark JSON object")
|
|
80
|
+
required = ["input_lens", "output_lens", "ttfts", "errors"]
|
|
81
|
+
required += ["latencies"] if schema == "vllm-bench-serve" else ["itls"]
|
|
82
|
+
if any(not isinstance(data.get(key), list) for key in required):
|
|
83
|
+
raise ValueError("detailed per-request arrays are required; aggregate percentiles are insufficient")
|
|
84
|
+
count = len(data["input_lens"])
|
|
85
|
+
if not count or any(len(data[key]) != count for key in required):
|
|
86
|
+
raise ValueError("per-request arrays must be nonempty and aligned")
|
|
87
|
+
optional = ["queue_times", "server_queue_times", "server_ttfts"]
|
|
88
|
+
for key in optional:
|
|
89
|
+
if key in data and (not isinstance(data[key], list) or len(data[key]) != count):
|
|
90
|
+
raise ValueError(f"{key} must align with the request arrays")
|
|
91
|
+
rows = []
|
|
92
|
+
for index in range(count):
|
|
93
|
+
error = data["errors"][index]
|
|
94
|
+
if not isinstance(error, str):
|
|
95
|
+
raise ValueError("each error must be a string")
|
|
96
|
+
completed = error == "" and data["output_lens"][index] > 0
|
|
97
|
+
latency = None
|
|
98
|
+
if completed:
|
|
99
|
+
if schema == "vllm-bench-serve":
|
|
100
|
+
latency = data["latencies"][index]
|
|
101
|
+
else:
|
|
102
|
+
gaps = np.asarray(data["itls"][index], dtype=float)
|
|
103
|
+
if gaps.ndim != 1 or not np.all(np.isfinite(gaps)) or np.any(gaps < 0):
|
|
104
|
+
raise ValueError("stream gaps must be finite nonnegative vectors")
|
|
105
|
+
latency = float(data["ttfts"][index] + gaps.sum())
|
|
106
|
+
timings = {key: data[key][index] if completed and key in data else None for key in optional}
|
|
107
|
+
rows.append(
|
|
108
|
+
ServingRequest(
|
|
109
|
+
input_tokens=data["input_lens"][index],
|
|
110
|
+
output_tokens=data["output_lens"][index],
|
|
111
|
+
status="completed" if completed else "failed",
|
|
112
|
+
ttft=data["ttfts"][index] if completed else None,
|
|
113
|
+
latency=latency,
|
|
114
|
+
client_queue_time=timings["queue_times"],
|
|
115
|
+
server_queue_time=timings["server_queue_times"],
|
|
116
|
+
server_ttft=timings["server_ttfts"],
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
successes = sum(row.status == "completed" for row in rows)
|
|
120
|
+
if data.get("completed") != successes or ("failed" in data and data["failed"] != count - successes):
|
|
121
|
+
raise ValueError("aggregate completion/failure counts disagree with detailed outcomes")
|
|
122
|
+
if "num_prompts" in data and data["num_prompts"] != count:
|
|
123
|
+
raise ValueError("num_prompts disagrees with offered cohort; do not silently drop requests")
|
|
124
|
+
metadata = {
|
|
125
|
+
key: data[key]
|
|
126
|
+
for key in ["model_id", "backend", "max_concurrency", "request_rate", "dataset_name"]
|
|
127
|
+
if key in data
|
|
128
|
+
}
|
|
129
|
+
metadata.update(schema=schema, latency_basis="client_latency" if schema == "vllm-bench-serve" else "stream_span")
|
|
130
|
+
if metadata.get("request_rate") == math.inf:
|
|
131
|
+
metadata["request_rate"] = "inf"
|
|
132
|
+
return ServingBenchmark(tuple(rows), data["duration"], metadata)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def read_vllm_benchmark(source, *, measurement_profile=None):
|
|
136
|
+
"""Read vLLM detailed JSON; preserve absent server metrics as None.
|
|
137
|
+
|
|
138
|
+
The caller must retain the producer version and hardware provenance. This
|
|
139
|
+
is a schema adapter, not detection of the engine/version from latency data.
|
|
140
|
+
"""
|
|
141
|
+
result = _read_arrays(source, "vllm-bench-serve")
|
|
142
|
+
result.metadata["measurement_profile"] = measurement_profile
|
|
143
|
+
return result
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def read_sglang_stream_benchmark(source, *, measurement_profile=None):
|
|
147
|
+
"""Read detailed streaming arrays with TTFT + sum(ITL) as observed span.
|
|
148
|
+
|
|
149
|
+
The span ends at the last output event, not necessarily HTTP completion.
|
|
150
|
+
ITL entries can contain multiple tokens; TPOT uses actual output_tokens.
|
|
151
|
+
"""
|
|
152
|
+
result = _read_arrays(source, "sglang-stream-arrays")
|
|
153
|
+
result.metadata["measurement_profile"] = measurement_profile
|
|
154
|
+
return result
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _quantiles(values):
|
|
158
|
+
if not values:
|
|
159
|
+
return None
|
|
160
|
+
return {
|
|
161
|
+
"count": len(values),
|
|
162
|
+
"mean": float(np.mean(values)),
|
|
163
|
+
**dict(zip(["p50", "p90", "p99"], np.percentile(values, [50, 90, 99]).tolist())),
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def summarize_serving_slo(benchmark, *, ttft_limit, tpot_limit):
|
|
168
|
+
"""Joint request SLO with all offered outcomes and pending-outcome bounds.
|
|
169
|
+
|
|
170
|
+
Completed one-token replies pass the TPOT condition by the vLLM goodput
|
|
171
|
+
convention but are excluded from TPOT means/quantiles. Pending requests
|
|
172
|
+
are unknown unless their already-observed TTFT violates its limit.
|
|
173
|
+
Counts/quantiles describe this cohort, without an iid confidence claim.
|
|
174
|
+
"""
|
|
175
|
+
if any(not math.isfinite(value) or value < 0 for value in (ttft_limit, tpot_limit)):
|
|
176
|
+
raise ValueError("SLO limits must be finite nonnegative seconds")
|
|
177
|
+
counts = {status: 0 for status in ["completed", "failed", "cancelled", "pending"]}
|
|
178
|
+
passed, unknown, single_token = 0, 0, 0
|
|
179
|
+
ttfts, tpots = [], []
|
|
180
|
+
for row in benchmark.requests:
|
|
181
|
+
counts[row.status] += 1
|
|
182
|
+
if row.status == "completed":
|
|
183
|
+
ttfts.append(row.ttft)
|
|
184
|
+
if row.tpot is None:
|
|
185
|
+
single_token += 1
|
|
186
|
+
else:
|
|
187
|
+
tpots.append(row.tpot)
|
|
188
|
+
passed += row.ttft <= ttft_limit and (row.tpot is None or row.tpot <= tpot_limit)
|
|
189
|
+
elif row.status == "pending" and (row.ttft is None or row.ttft <= ttft_limit):
|
|
190
|
+
unknown += 1
|
|
191
|
+
offered = len(benchmark.requests)
|
|
192
|
+
return {
|
|
193
|
+
"offered": offered,
|
|
194
|
+
**counts,
|
|
195
|
+
"single_token_completed": single_token,
|
|
196
|
+
"joint_passed": passed,
|
|
197
|
+
"joint_unknown": unknown,
|
|
198
|
+
"joint_fraction_bounds": [passed / offered, (passed + unknown) / offered],
|
|
199
|
+
"observed_goodput": passed / benchmark.duration,
|
|
200
|
+
"duration": benchmark.duration,
|
|
201
|
+
"ttft_limit": ttft_limit,
|
|
202
|
+
"tpot_limit": tpot_limit,
|
|
203
|
+
"completed_ttft_seconds": _quantiles(ttfts),
|
|
204
|
+
"completed_multitoken_tpot_seconds": _quantiles(tpots),
|
|
205
|
+
"server_queue_observed": sum(row.server_queue_time is not None for row in benchmark.requests),
|
|
206
|
+
"server_ttft_observed": sum(row.server_ttft is not None for row in benchmark.requests),
|
|
207
|
+
"latency_basis": benchmark.metadata.get("latency_basis", "explicit_request_records"),
|
|
208
|
+
"uncertainty": "pending-outcome bounds; no iid confidence interval",
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def from_prefill_decode(result, arrivals, inputs, outputs, *, horizon=None):
|
|
213
|
+
"""Convert token DES timestamps; optionally censor at a fixed horizon.
|
|
214
|
+
|
|
215
|
+
Future requests are outside the offered cohort. Future completions and
|
|
216
|
+
tokens are hidden rather than counted as successful outcomes at the cut.
|
|
217
|
+
No steady-state or stationarity assertion is made for the drained cohort.
|
|
218
|
+
"""
|
|
219
|
+
arrivals = np.asarray(arrivals, dtype=float)
|
|
220
|
+
if arrivals.ndim != 1 or not arrivals.size or not np.all(np.isfinite(arrivals)) or np.any(np.diff(arrivals) < 0):
|
|
221
|
+
raise ValueError("arrivals must be nonempty finite chronological times")
|
|
222
|
+
if any(len(values) != len(arrivals) for values in [inputs, outputs, result.output_times, result.completion_times]):
|
|
223
|
+
raise ValueError("DES arrays must align")
|
|
224
|
+
for lengths in (inputs, outputs):
|
|
225
|
+
values = np.asarray(lengths, dtype=float)
|
|
226
|
+
if not np.all(np.isfinite(values)) or np.any(values <= 0) or np.any(values != np.floor(values)):
|
|
227
|
+
raise ValueError("DES token lengths must be positive integers")
|
|
228
|
+
for arrival, count, stamps, completion in zip(arrivals, outputs, result.output_times, result.completion_times):
|
|
229
|
+
stamps = np.asarray(stamps, dtype=float)
|
|
230
|
+
if stamps.ndim != 1 or stamps.size != count or not np.all(np.isfinite(stamps)):
|
|
231
|
+
raise ValueError("DES token timestamps must align with output lengths and be finite")
|
|
232
|
+
if (
|
|
233
|
+
np.any(np.diff(stamps) < 0)
|
|
234
|
+
or stamps[0] < arrival
|
|
235
|
+
or not math.isfinite(completion)
|
|
236
|
+
or stamps[-1] != completion
|
|
237
|
+
):
|
|
238
|
+
raise ValueError("DES token timestamps, output lengths and completion must agree")
|
|
239
|
+
end = max(result.completion_times) if horizon is None else horizon
|
|
240
|
+
if not math.isfinite(end) or end <= arrivals[0]:
|
|
241
|
+
raise ValueError("observation horizon must follow the first arrival")
|
|
242
|
+
rows = []
|
|
243
|
+
for index, arrival in enumerate(arrivals):
|
|
244
|
+
if arrival > end:
|
|
245
|
+
break
|
|
246
|
+
tokens = result.output_times[index]
|
|
247
|
+
observed = [stamp for stamp in tokens if stamp <= end]
|
|
248
|
+
completed = result.completion_times[index] <= end
|
|
249
|
+
rows.append(
|
|
250
|
+
ServingRequest(
|
|
251
|
+
input_tokens=int(inputs[index]),
|
|
252
|
+
output_tokens=int(outputs[index]) if completed else len(observed),
|
|
253
|
+
status="completed" if completed else "pending",
|
|
254
|
+
ttft=tokens[0] - arrival if observed else None,
|
|
255
|
+
latency=result.completion_times[index] - arrival if completed else None,
|
|
256
|
+
)
|
|
257
|
+
)
|
|
258
|
+
return ServingBenchmark(tuple(rows), float(end - arrivals[0]), {"latency_basis": "token_DES"})
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""History-only conditional ECDFs with audited fallback and optional scale ratios.
|
|
2
|
+
|
|
3
|
+
CRPS uses the empirical energy identity, Gneiting & Raftery (2007),
|
|
4
|
+
https://doi.org/10.1198/016214506000001437. It scores a full predictive CDF,
|
|
5
|
+
not a random sample from it. Feature availability must be established by callers.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections import defaultdict
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from numbers import Integral, Real
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from most_queue.random.service_calibration import ServiceCalibration
|
|
15
|
+
from most_queue.random.trace_resampling import ConditionalEmpirical
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _positive(value, name):
|
|
19
|
+
if isinstance(value, (bool, np.bool_)) or not isinstance(value, Real) or not np.isfinite(value) or value <= 0:
|
|
20
|
+
raise ValueError(f"{name} must be positive and finite")
|
|
21
|
+
return float(value)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _need(value):
|
|
25
|
+
if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral) or value < 1:
|
|
26
|
+
raise ValueError("need must be a positive integer")
|
|
27
|
+
return int(value)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _context(value):
|
|
31
|
+
if value is not None and (not isinstance(value, str) or not value):
|
|
32
|
+
raise ValueError("context must be a nonempty string or None")
|
|
33
|
+
return value
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def request_bucket(request):
|
|
37
|
+
"""Fixed right-closed request bins in seconds; unknown is distinct from zero."""
|
|
38
|
+
if request is None:
|
|
39
|
+
return None
|
|
40
|
+
value = _positive(request, "request")
|
|
41
|
+
return str(int(np.searchsorted([300, 1800, 7200, 28800, 86400], value)))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class EmpiricalPrediction:
|
|
46
|
+
"""An immutable selected ECDF, positive target scale and audited fit level."""
|
|
47
|
+
|
|
48
|
+
distribution: ServiceCalibration
|
|
49
|
+
scale: float = 1.0
|
|
50
|
+
level: str = "pooled"
|
|
51
|
+
|
|
52
|
+
def __post_init__(self):
|
|
53
|
+
_positive(self.scale, "scale")
|
|
54
|
+
# Check the support before sampling, not only the selected quantile.
|
|
55
|
+
_positive(self.scale * self.distribution.samples[-1], "scaled support")
|
|
56
|
+
_positive(self.scale * self.distribution.samples[0], "scaled support")
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def mean(self):
|
|
60
|
+
"""Predictive mean in service-time units, not ratio units."""
|
|
61
|
+
return self.scale * self.distribution.mean
|
|
62
|
+
|
|
63
|
+
def quantile(self, probability):
|
|
64
|
+
"""Inverse discrete ECDF; no interpolation or truncation at runtime budget."""
|
|
65
|
+
if isinstance(probability, (bool, np.bool_)) or not isinstance(probability, Real):
|
|
66
|
+
raise ValueError("probability must be a finite real in (0,1)")
|
|
67
|
+
return float(self.scale * self.distribution.quantiles("empirical", [probability])[0])
|
|
68
|
+
|
|
69
|
+
def probability_exceeding(self, threshold):
|
|
70
|
+
"""Return P(S > threshold), with equality counted as within the budget."""
|
|
71
|
+
threshold = _positive(threshold, "threshold")
|
|
72
|
+
support = np.array(self.distribution.samples) * self.scale
|
|
73
|
+
return float(np.mean(support > threshold))
|
|
74
|
+
|
|
75
|
+
def crps(self, observed):
|
|
76
|
+
"""Compute E|X-y| - .5 E|X-X'| exactly for a finite equal-weight ECDF.
|
|
77
|
+
|
|
78
|
+
For sorted support x_i the second term is sum((2i-n-1)x_i)/n².
|
|
79
|
+
Scaling before evaluation keeps score units in seconds for ratio fits.
|
|
80
|
+
"""
|
|
81
|
+
observed = _positive(observed, "observed")
|
|
82
|
+
support = np.asarray(self.distribution.samples) * self.scale
|
|
83
|
+
n = len(support)
|
|
84
|
+
dispersion = np.dot((2 * np.arange(1, n + 1) - n - 1) / n, support) / n
|
|
85
|
+
score = float(np.mean(np.abs(support - observed)) - dispersion)
|
|
86
|
+
if not np.isfinite(score):
|
|
87
|
+
raise ValueError("CRPS overflow; rescale observations")
|
|
88
|
+
return max(0.0, score)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class FeatureConditionalEmpirical:
|
|
93
|
+
"""Joint context/need cells, then coarse/pooled fallback; fit uses history only.
|
|
94
|
+
|
|
95
|
+
Optional scales fit service/scale on known positive scales, multiplying by
|
|
96
|
+
the supplied target scale at prediction. Unknown target scale or absent
|
|
97
|
+
ratio history falls back to absolute service. No outcome or test S is used.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
absolute: ConditionalEmpirical
|
|
101
|
+
pooled: ServiceCalibration | None
|
|
102
|
+
coarse: dict
|
|
103
|
+
joint: dict
|
|
104
|
+
exact_joint: dict
|
|
105
|
+
ratio: bool
|
|
106
|
+
audit: dict
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def fit(
|
|
110
|
+
cls, needs, services, contexts, *, scales=None, exact=False, minimum=20
|
|
111
|
+
): # pylint: disable=too-many-arguments
|
|
112
|
+
"""Fit context ECDFs with a fixed minimum count and explicit missing scales."""
|
|
113
|
+
needs, contexts = tuple(needs), tuple(contexts)
|
|
114
|
+
if len(contexts) != len(needs):
|
|
115
|
+
raise ValueError("contexts must match needs")
|
|
116
|
+
needs = tuple(_need(value) for value in needs)
|
|
117
|
+
contexts = tuple(_context(value) for value in contexts)
|
|
118
|
+
absolute = ConditionalEmpirical.fit(needs, services, minimum=minimum)
|
|
119
|
+
if not isinstance(exact, bool):
|
|
120
|
+
raise ValueError("exact must be boolean")
|
|
121
|
+
values = np.asarray(services, dtype=float)
|
|
122
|
+
ratio = scales is not None
|
|
123
|
+
scales = tuple(scales) if ratio else (1.0,) * len(needs)
|
|
124
|
+
if len(scales) != len(needs):
|
|
125
|
+
raise ValueError("scales must match needs")
|
|
126
|
+
pools = {"coarse": defaultdict(list), "joint": defaultdict(list), "exact": defaultdict(list)}
|
|
127
|
+
normalized = []
|
|
128
|
+
for need, service, context, scale in zip(needs, values, contexts, scales):
|
|
129
|
+
if scale is None:
|
|
130
|
+
continue
|
|
131
|
+
value = _positive(service / _positive(scale, "scale"), "scaled service")
|
|
132
|
+
group = int(np.searchsorted([1, 8, 32], need))
|
|
133
|
+
normalized.append(value)
|
|
134
|
+
pools["coarse"][group].append(value)
|
|
135
|
+
pools["joint"][group, context].append(value)
|
|
136
|
+
if exact:
|
|
137
|
+
pools["exact"][need, context].append(value)
|
|
138
|
+
fitted = {
|
|
139
|
+
name: {key: ServiceCalibration.fit(sample) for key, sample in cells.items() if len(sample) >= minimum}
|
|
140
|
+
for name, cells in pools.items()
|
|
141
|
+
}
|
|
142
|
+
pooled = ServiceCalibration.fit(normalized) if len(normalized) >= 2 else None
|
|
143
|
+
audit = {
|
|
144
|
+
"history_count": len(needs),
|
|
145
|
+
"scaled_count": len(normalized),
|
|
146
|
+
"missing_scale_count": sum(s is None for s in scales),
|
|
147
|
+
"minimum": int(minimum),
|
|
148
|
+
"exact": exact,
|
|
149
|
+
"ratio": ratio,
|
|
150
|
+
"fitted_cells": {name: len(cells) for name, cells in fitted.items()},
|
|
151
|
+
}
|
|
152
|
+
return cls(absolute, pooled, fitted["coarse"], fitted["joint"], fitted["exact"], ratio, audit)
|
|
153
|
+
|
|
154
|
+
def predict(self, need, context, *, scale=None):
|
|
155
|
+
"""Return the complete predictive distribution and the fallback used."""
|
|
156
|
+
need, context = _need(need), _context(context)
|
|
157
|
+
if scale is not None:
|
|
158
|
+
_positive(scale, "scale")
|
|
159
|
+
if self.ratio and (scale is None or self.pooled is None):
|
|
160
|
+
distribution, level = self.absolute.distribution(need)
|
|
161
|
+
return EmpiricalPrediction(distribution, 1.0, f"absolute_{level}")
|
|
162
|
+
if not self.ratio and scale is not None:
|
|
163
|
+
raise ValueError("scale is only valid for a ratio fit")
|
|
164
|
+
factor = float(scale) if self.ratio else 1.0
|
|
165
|
+
group = int(np.searchsorted([1, 8, 32], need))
|
|
166
|
+
choices = (
|
|
167
|
+
(self.exact_joint.get((need, context)), "exact_context"),
|
|
168
|
+
(self.joint.get((group, context)), "coarse_context"),
|
|
169
|
+
(self.coarse.get(group), "coarse"),
|
|
170
|
+
(self.pooled, "pooled"),
|
|
171
|
+
)
|
|
172
|
+
for distribution, level in choices:
|
|
173
|
+
if distribution is not None:
|
|
174
|
+
return EmpiricalPrediction(distribution, factor, level)
|
|
175
|
+
raise ValueError("no predictive distribution available")
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Empirical marked arrivals with zero gaps and explicit matched controls.
|
|
2
|
+
|
|
3
|
+
Circular blocks use the end-to-start convention documented at
|
|
4
|
+
https://bashtage.github.io/arch/bootstrap/generated/arch.bootstrap.CircularBlockBootstrap.html.
|
|
5
|
+
This is a workload generator, not a stationarity assumption or population CI.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, replace
|
|
9
|
+
from numbers import Integral, Real
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from most_queue.random.trace_resampling import circular_block_indices
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _count(value, name, minimum=1):
|
|
17
|
+
if isinstance(value, (bool, np.bool_)) or not isinstance(value, Integral) or value < minimum:
|
|
18
|
+
raise ValueError(f"{name} must be an integer >= {minimum}")
|
|
19
|
+
return int(value)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _real(value, name, minimum=0):
|
|
23
|
+
if isinstance(value, (bool, np.bool_)) or not isinstance(value, Real) or not np.isfinite(value) or value < minimum:
|
|
24
|
+
raise ValueError(f"{name} must be finite and >= {minimum}")
|
|
25
|
+
return float(value)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class ArrivalMark:
|
|
30
|
+
"""A preceding gap in seconds, resource demand, and a coupled feature bundle."""
|
|
31
|
+
|
|
32
|
+
gap: float
|
|
33
|
+
need: int
|
|
34
|
+
context: str | None = None
|
|
35
|
+
requested_time: float | None = None
|
|
36
|
+
|
|
37
|
+
def __post_init__(self):
|
|
38
|
+
_real(self.gap, "gap")
|
|
39
|
+
_count(self.need, "need")
|
|
40
|
+
if self.context is not None and (not isinstance(self.context, str) or not self.context):
|
|
41
|
+
raise ValueError("context must be a nonempty string or None")
|
|
42
|
+
if self.requested_time is not None:
|
|
43
|
+
_real(self.requested_time, "requested_time")
|
|
44
|
+
if self.requested_time == 0:
|
|
45
|
+
raise ValueError("requested_time must be positive or None")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class MarkedArrivalBootstrap:
|
|
50
|
+
"""Immutable donor tuples; callers must supply a temporally valid history."""
|
|
51
|
+
|
|
52
|
+
donors: tuple[ArrivalMark, ...]
|
|
53
|
+
|
|
54
|
+
def __post_init__(self):
|
|
55
|
+
object.__setattr__(self, "donors", tuple(self.donors))
|
|
56
|
+
if not self.donors or any(not isinstance(item, ArrivalMark) for item in self.donors):
|
|
57
|
+
raise ValueError("donors must be nonempty ArrivalMark records")
|
|
58
|
+
|
|
59
|
+
@classmethod
|
|
60
|
+
def fit(cls, arrivals, needs, contexts, requested_times=None):
|
|
61
|
+
"""Form true adjacent gaps; the first row supplies only a predecessor.
|
|
62
|
+
|
|
63
|
+
Equal timestamps are retained without jitter. Filtering history before
|
|
64
|
+
calling this method changes adjacency and is the caller's responsibility.
|
|
65
|
+
"""
|
|
66
|
+
arrivals, needs, contexts = tuple(arrivals), tuple(needs), tuple(contexts)
|
|
67
|
+
requests = (None,) * len(arrivals) if requested_times is None else tuple(requested_times)
|
|
68
|
+
if len(arrivals) < 2 or not len(arrivals) == len(needs) == len(contexts) == len(requests):
|
|
69
|
+
raise ValueError("history fields must match and contain at least two rows")
|
|
70
|
+
times = np.array([_real(value, "arrival") for value in arrivals])
|
|
71
|
+
gaps = np.diff(times)
|
|
72
|
+
if np.any(gaps < 0):
|
|
73
|
+
raise ValueError("arrivals must be chronological")
|
|
74
|
+
# Validate the predecessor's fields too; it is not itself a donor.
|
|
75
|
+
records = tuple(
|
|
76
|
+
ArrivalMark(float(gap), need, context, request)
|
|
77
|
+
for gap, need, context, request in zip(np.r_[0, gaps], needs, contexts, requests)
|
|
78
|
+
)
|
|
79
|
+
return cls(records[1:])
|
|
80
|
+
|
|
81
|
+
def sample(self, uniforms, *, block_length=1):
|
|
82
|
+
"""Select uniform donor starts, wrap fixed blocks, and truncate to size."""
|
|
83
|
+
indices = circular_block_indices(len(self.donors), uniforms, block_length)
|
|
84
|
+
return tuple(self.donors[int(index)] for index in indices)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def anchored_permutation(uniforms, warmup):
|
|
88
|
+
"""Shuffle each cohort partition with its first row fixed, using stable keys.
|
|
89
|
+
|
|
90
|
+
Keeping rows 0 and warmup fixed preserves the first arrival and measured
|
|
91
|
+
arrival horizon when whole gap/mark/service tuples move together.
|
|
92
|
+
"""
|
|
93
|
+
values = np.asarray(uniforms)
|
|
94
|
+
if values.ndim != 1 or not values.size or values.dtype.kind not in "iuf" or not np.all(np.isfinite(values)):
|
|
95
|
+
raise ValueError("uniform keys must be a nonempty finite numeric vector")
|
|
96
|
+
if np.any((values <= 0) | (values >= 1)):
|
|
97
|
+
raise ValueError("uniform keys must lie strictly inside (0,1)")
|
|
98
|
+
warmup = _count(warmup, "warmup", minimum=0)
|
|
99
|
+
if warmup >= len(values):
|
|
100
|
+
raise ValueError("warmup must leave measured jobs")
|
|
101
|
+
order = np.arange(len(values))
|
|
102
|
+
for begin, end in ((0, warmup), (warmup, len(values))):
|
|
103
|
+
if end > begin + 1:
|
|
104
|
+
order[begin + 1 : end] = begin + 1 + np.argsort(values[begin + 1 : end], kind="stable")
|
|
105
|
+
return order
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def decouple_gaps(records, order):
|
|
109
|
+
"""Retain each gap while moving coupled need/context/request bundles."""
|
|
110
|
+
records, order = tuple(records), np.asarray(order)
|
|
111
|
+
if order.ndim != 1 or order.dtype.kind not in "iu" or sorted(order.tolist()) != list(range(len(records))):
|
|
112
|
+
raise ValueError("order must be a complete permutation of records")
|
|
113
|
+
if not records or any(not isinstance(item, ArrivalMark) for item in records):
|
|
114
|
+
raise ValueError("records must be nonempty ArrivalMark records")
|
|
115
|
+
return tuple(replace(records[int(index)], gap=record.gap) for record, index in zip(records, order))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def arrival_times(records):
|
|
119
|
+
"""Accumulate nonnegative gaps, anchoring the first job at zero."""
|
|
120
|
+
records = tuple(records)
|
|
121
|
+
if not records or any(not isinstance(item, ArrivalMark) for item in records):
|
|
122
|
+
raise ValueError("records must be nonempty ArrivalMark records")
|
|
123
|
+
with np.errstate(over="ignore"):
|
|
124
|
+
values = np.cumsum([record.gap for record in records]) - records[0].gap
|
|
125
|
+
if not np.all(np.isfinite(values)):
|
|
126
|
+
raise ValueError("cumulative arrival timestamps overflow")
|
|
127
|
+
return values
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Select a service model using positive queue summaries from validation only.
|
|
2
|
+
|
|
3
|
+
The mean absolute log error is an explicitly chosen downstream objective,
|
|
4
|
+
not a proper distributional scoring rule. Callers own temporal separation,
|
|
5
|
+
cohort comparability and aggregation over Monte Carlo replications.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _metrics(values):
|
|
14
|
+
array = np.asarray(values)
|
|
15
|
+
if array.dtype.kind not in "iuf" or array.size == 0 or not np.all(np.isfinite(array)) or np.any(array <= 0):
|
|
16
|
+
raise ValueError("queue metrics must be nonempty, finite, positive real values")
|
|
17
|
+
return array.astype(float)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def queue_log_error(predicted, observed):
|
|
21
|
+
"""Mean |log(predicted)-log(observed)| with equal weights and exact shapes.
|
|
22
|
+
|
|
23
|
+
Pass MC means when comparing expected queue summaries. Zero references are
|
|
24
|
+
rejected, never stabilized with an implicit epsilon. Log differences avoid
|
|
25
|
+
overflow in a ratio of valid finite metrics.
|
|
26
|
+
"""
|
|
27
|
+
predicted, observed = _metrics(predicted), _metrics(observed)
|
|
28
|
+
if predicted.shape != observed.shape:
|
|
29
|
+
raise ValueError("predicted and observed shapes must match exactly")
|
|
30
|
+
return float(np.mean(np.abs(np.log(predicted) - np.log(observed))))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def select_queue_model(predictions, observed, *, candidate_order):
|
|
34
|
+
"""Score supplied validation summaries; break exact ties in declared order.
|
|
35
|
+
|
|
36
|
+
Predictions maps every candidate name to an array with the observed shape.
|
|
37
|
+
This function neither fits a model nor reads held-out outcomes.
|
|
38
|
+
"""
|
|
39
|
+
if isinstance(candidate_order, (str, bytes)):
|
|
40
|
+
raise ValueError("candidate_order must be a sequence of unique names")
|
|
41
|
+
order = tuple(candidate_order)
|
|
42
|
+
if not order or any(not isinstance(name, str) or not name for name in order) or len(set(order)) != len(order):
|
|
43
|
+
raise ValueError("candidate_order must contain unique nonempty names")
|
|
44
|
+
if not isinstance(predictions, Mapping) or set(predictions) != set(order):
|
|
45
|
+
raise ValueError("predictions must contain exactly the declared candidates")
|
|
46
|
+
losses = {name: queue_log_error(predictions[name], observed) for name in order}
|
|
47
|
+
return {"selected": min(order, key=losses.get), "losses": losses, "candidate_order": list(order)}
|