vtune 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vtune-0.1.0a1/LICENSE +21 -0
- vtune-0.1.0a1/PKG-INFO +154 -0
- vtune-0.1.0a1/README.md +132 -0
- vtune-0.1.0a1/pyproject.toml +33 -0
- vtune-0.1.0a1/setup.cfg +4 -0
- vtune-0.1.0a1/src/vtune/__init__.py +7 -0
- vtune-0.1.0a1/src/vtune/benchmarks/__init__.py +9 -0
- vtune-0.1.0a1/src/vtune/benchmarks/guidellm.py +145 -0
- vtune-0.1.0a1/src/vtune/benchmarks/timing.py +67 -0
- vtune-0.1.0a1/src/vtune/cli.py +86 -0
- vtune-0.1.0a1/src/vtune/cli_options.py +38 -0
- vtune-0.1.0a1/src/vtune/config/__init__.py +6 -0
- vtune-0.1.0a1/src/vtune/config/errors.py +17 -0
- vtune-0.1.0a1/src/vtune/config/loader.py +147 -0
- vtune-0.1.0a1/src/vtune/config/models.py +42 -0
- vtune-0.1.0a1/src/vtune/config/runtime.py +56 -0
- vtune-0.1.0a1/src/vtune/domain/__init__.py +25 -0
- vtune-0.1.0a1/src/vtune/domain/attempt_report.py +18 -0
- vtune-0.1.0a1/src/vtune/domain/benchmark.py +46 -0
- vtune-0.1.0a1/src/vtune/domain/models.py +69 -0
- vtune-0.1.0a1/src/vtune/domain/results.py +59 -0
- vtune-0.1.0a1/src/vtune/domain/states.py +31 -0
- vtune-0.1.0a1/src/vtune/domain/trial_report.py +70 -0
- vtune-0.1.0a1/src/vtune/lifecycle/__init__.py +5 -0
- vtune-0.1.0a1/src/vtune/lifecycle/integrity.py +102 -0
- vtune-0.1.0a1/src/vtune/lifecycle/retry.py +106 -0
- vtune-0.1.0a1/src/vtune/managers/__init__.py +9 -0
- vtune-0.1.0a1/src/vtune/managers/results.py +65 -0
- vtune-0.1.0a1/src/vtune/managers/run_results.py +114 -0
- vtune-0.1.0a1/src/vtune/managers/run_session.py +69 -0
- vtune-0.1.0a1/src/vtune/managers/scoring.py +57 -0
- vtune-0.1.0a1/src/vtune/managers/trial.py +97 -0
- vtune-0.1.0a1/src/vtune/orchestrator.py +150 -0
- vtune-0.1.0a1/src/vtune/py.typed +1 -0
- vtune-0.1.0a1/src/vtune/reporting/__init__.py +5 -0
- vtune-0.1.0a1/src/vtune/reporting/analysis.py +80 -0
- vtune-0.1.0a1/src/vtune/reporting/charts.py +94 -0
- vtune-0.1.0a1/src/vtune/reporting/context.py +17 -0
- vtune-0.1.0a1/src/vtune/reporting/dashboard.py +113 -0
- vtune-0.1.0a1/src/vtune/reporting/reporter.py +57 -0
- vtune-0.1.0a1/src/vtune/reporting/tables.py +91 -0
- vtune-0.1.0a1/src/vtune/reproduction/__init__.py +1 -0
- vtune-0.1.0a1/src/vtune/reproduction/display.py +63 -0
- vtune-0.1.0a1/src/vtune/reproduction/export.py +13 -0
- vtune-0.1.0a1/src/vtune/reproduction/manifest.py +72 -0
- vtune-0.1.0a1/src/vtune/reproduction/metadata.py +65 -0
- vtune-0.1.0a1/src/vtune/reproduction/models.py +36 -0
- vtune-0.1.0a1/src/vtune/reproduction/reader.py +48 -0
- vtune-0.1.0a1/src/vtune/reproduction/redaction.py +39 -0
- vtune-0.1.0a1/src/vtune/search/__init__.py +6 -0
- vtune-0.1.0a1/src/vtune/search/factory.py +40 -0
- vtune-0.1.0a1/src/vtune/search/fixed_session.py +26 -0
- vtune-0.1.0a1/src/vtune/search/grid.py +65 -0
- vtune-0.1.0a1/src/vtune/search/grid_session.py +29 -0
- vtune-0.1.0a1/src/vtune/search/optuna_session.py +121 -0
- vtune-0.1.0a1/src/vtune/search/strategy.py +18 -0
- vtune-0.1.0a1/src/vtune/terminal.py +30 -0
- vtune-0.1.0a1/src/vtune/workers/__init__.py +21 -0
- vtune-0.1.0a1/src/vtune/workers/attempts.py +15 -0
- vtune-0.1.0a1/src/vtune/workers/base.py +32 -0
- vtune-0.1.0a1/src/vtune/workers/benchmark.py +98 -0
- vtune-0.1.0a1/src/vtune/workers/configuration.py +105 -0
- vtune-0.1.0a1/src/vtune/workers/factory.py +43 -0
- vtune-0.1.0a1/src/vtune/workers/failure_details.py +40 -0
- vtune-0.1.0a1/src/vtune/workers/process.py +147 -0
- vtune-0.1.0a1/src/vtune/workers/readiness.py +116 -0
- vtune-0.1.0a1/src/vtune/workers/vllm.py +67 -0
- vtune-0.1.0a1/src/vtune.egg-info/PKG-INFO +154 -0
- vtune-0.1.0a1/src/vtune.egg-info/SOURCES.txt +71 -0
- vtune-0.1.0a1/src/vtune.egg-info/dependency_links.txt +1 -0
- vtune-0.1.0a1/src/vtune.egg-info/entry_points.txt +2 -0
- vtune-0.1.0a1/src/vtune.egg-info/requires.txt +2 -0
- vtune-0.1.0a1/src/vtune.egg-info/top_level.txt +1 -0
vtune-0.1.0a1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 vTune contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
vtune-0.1.0a1/PKG-INFO
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vtune
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Local-first experimentation for vLLM serving configurations
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/brtydse100/vTune
|
|
7
|
+
Project-URL: Issues, https://github.com/brtydse100/vTune/issues
|
|
8
|
+
Keywords: vllm,guidellm,benchmark,optimization,inference
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: Optuna<5,>=4.0
|
|
20
|
+
Requires-Dist: PyYAML>=6.0
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# vTune
|
|
24
|
+
|
|
25
|
+
vTune is a local-first experimentation and optimization tool for vLLM serving
|
|
26
|
+
configurations. Users define the parameters and workloads they care about;
|
|
27
|
+
vTune manages the server lifecycle, runs repeatable benchmarks, explores the
|
|
28
|
+
search space, and reports which configurations performed best.
|
|
29
|
+
|
|
30
|
+
vTune is alpha software targeting Linux with NVIDIA GPUs and Python 3.11–3.12.
|
|
31
|
+
|
|
32
|
+
Tested integrations:
|
|
33
|
+
|
|
34
|
+
- vLLM 0.10.2 with GuideLLM 0.7.3 on WSL2 and an RTX 3080.
|
|
35
|
+
- vLLM 0.28.0 with GuideLLM 0.7.3 on the same system. WSL2 required
|
|
36
|
+
`VLLM_USE_V2_MODEL_RUNNER: "0"` because UVA was unavailable and
|
|
37
|
+
`VLLM_USE_FLASHINFER_SAMPLER: "0"` because the CUDA compiler toolkit was
|
|
38
|
+
not installed. Native Linux systems may not require these settings.
|
|
39
|
+
|
|
40
|
+
Other combinations may work but are not yet verified.
|
|
41
|
+
|
|
42
|
+
The published `py3-none-any` wheel installs on Linux and Windows. Configuration
|
|
43
|
+
validation and stored-result inspection work on Windows, but starting an
|
|
44
|
+
experiment is supported only on Linux because vLLM has no native Windows
|
|
45
|
+
runtime.
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
Requirements: Linux, an NVIDIA GPU, a local model directory, and working
|
|
50
|
+
`vllm` and `guidellm` commands. Install this checkout with:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install -e .
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Create `experiment.yaml`:
|
|
57
|
+
|
|
58
|
+
```yaml
|
|
59
|
+
schema_version: 1
|
|
60
|
+
experiment:
|
|
61
|
+
name: first-run
|
|
62
|
+
model:
|
|
63
|
+
path: /models/opt-125m
|
|
64
|
+
server:
|
|
65
|
+
args:
|
|
66
|
+
gpu-memory-utilization: 0.8
|
|
67
|
+
tune:
|
|
68
|
+
max-num-seqs:
|
|
69
|
+
values: [8, 16]
|
|
70
|
+
benchmark:
|
|
71
|
+
runs:
|
|
72
|
+
- name: throughput
|
|
73
|
+
profile:
|
|
74
|
+
kind: throughput
|
|
75
|
+
max_concurrency: 16
|
|
76
|
+
constraints:
|
|
77
|
+
- kind: max_requests
|
|
78
|
+
count: 10
|
|
79
|
+
data:
|
|
80
|
+
- kind: synthetic_text
|
|
81
|
+
prompt_tokens: 32
|
|
82
|
+
output_tokens: 16
|
|
83
|
+
optimization:
|
|
84
|
+
maximize: output_tokens_per_second
|
|
85
|
+
sampler: tpe
|
|
86
|
+
trials: 2
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Run it:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
vtune --config experiment.yaml
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The short form is `vtune -c experiment.yaml`. The command validates the file,
|
|
96
|
+
runs the experiment, persists results, and generates its exports and report.
|
|
97
|
+
vTune binds vLLM to `127.0.0.1` by default. Set `server.args.host` explicitly
|
|
98
|
+
only when the benchmark server must be reachable from another host.
|
|
99
|
+
|
|
100
|
+
Terminal output is concise by default. To stream vLLM and GuideLLM logs:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
vtune --config experiment.yaml --verbose
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
The persistent equivalent uses GuideLLM's logging level names:
|
|
107
|
+
|
|
108
|
+
```yaml
|
|
109
|
+
logging:
|
|
110
|
+
level: DEBUG
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Supported levels are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.
|
|
114
|
+
Full per-trial log files are always saved. `--verbose` overrides the configured
|
|
115
|
+
level with `DEBUG` for that invocation.
|
|
116
|
+
|
|
117
|
+
Retry one or more selected trials into a new immutable linked run:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
vtune retry --run runs/EXPERIMENT/RUN_ID \
|
|
121
|
+
--trial trial-0001 --trial trial-0004
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
The source run is never modified.
|
|
125
|
+
|
|
126
|
+
Display every stored vLLM and GuideLLM command for a trial without executing
|
|
127
|
+
anything:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
vtune reproduce --run runs/EXPERIMENT/RUN_ID --trial trial-0001
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Each completed run also contains a self-contained `report.html` decision
|
|
134
|
+
dashboard with the best observed configuration, baseline comparison, score
|
|
135
|
+
history, throughput/latency tradeoff, and observed parameter effects.
|
|
136
|
+
|
|
137
|
+
Random and TPE runs never execute the same resolved configuration twice.
|
|
138
|
+
`optimization.trials` cannot exceed the number of unique configurations in
|
|
139
|
+
the declared search space.
|
|
140
|
+
|
|
141
|
+
## Product documents
|
|
142
|
+
|
|
143
|
+
- [First MVP specification](docs/MVP_SPEC.md)
|
|
144
|
+
- [Future implementation roadmap](docs/ROADMAP.md)
|
|
145
|
+
- [Architecture overview and early sketch](docs/ARCHITECTURE.md)
|
|
146
|
+
- [Editable Draw.io architecture diagram](docs/vtune-architecture.drawio)
|
|
147
|
+
- [Contributor guide](CONTRIBUTING.md)
|
|
148
|
+
- [Release notes](CHANGELOG.md)
|
|
149
|
+
|
|
150
|
+
The MVP specification defines the first releasable version and its acceptance
|
|
151
|
+
criteria. The roadmap describes capabilities that should be designed for now
|
|
152
|
+
but implemented after the core experiment loop is reliable.
|
|
153
|
+
|
|
154
|
+
vTune is available under the [MIT License](LICENSE).
|
vtune-0.1.0a1/README.md
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# vTune
|
|
2
|
+
|
|
3
|
+
vTune is a local-first experimentation and optimization tool for vLLM serving
|
|
4
|
+
configurations. Users define the parameters and workloads they care about;
|
|
5
|
+
vTune manages the server lifecycle, runs repeatable benchmarks, explores the
|
|
6
|
+
search space, and reports which configurations performed best.
|
|
7
|
+
|
|
8
|
+
vTune is alpha software targeting Linux with NVIDIA GPUs and Python 3.11–3.12.
|
|
9
|
+
|
|
10
|
+
Tested integrations:
|
|
11
|
+
|
|
12
|
+
- vLLM 0.10.2 with GuideLLM 0.7.3 on WSL2 and an RTX 3080.
|
|
13
|
+
- vLLM 0.28.0 with GuideLLM 0.7.3 on the same system. WSL2 required
|
|
14
|
+
`VLLM_USE_V2_MODEL_RUNNER: "0"` because UVA was unavailable and
|
|
15
|
+
`VLLM_USE_FLASHINFER_SAMPLER: "0"` because the CUDA compiler toolkit was
|
|
16
|
+
not installed. Native Linux systems may not require these settings.
|
|
17
|
+
|
|
18
|
+
Other combinations may work but are not yet verified.
|
|
19
|
+
|
|
20
|
+
The published `py3-none-any` wheel installs on Linux and Windows. Configuration
|
|
21
|
+
validation and stored-result inspection work on Windows, but starting an
|
|
22
|
+
experiment is supported only on Linux because vLLM has no native Windows
|
|
23
|
+
runtime.
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
Requirements: Linux, an NVIDIA GPU, a local model directory, and working
|
|
28
|
+
`vllm` and `guidellm` commands. Install this checkout with:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install -e .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Create `experiment.yaml`:
|
|
35
|
+
|
|
36
|
+
```yaml
|
|
37
|
+
schema_version: 1
|
|
38
|
+
experiment:
|
|
39
|
+
name: first-run
|
|
40
|
+
model:
|
|
41
|
+
path: /models/opt-125m
|
|
42
|
+
server:
|
|
43
|
+
args:
|
|
44
|
+
gpu-memory-utilization: 0.8
|
|
45
|
+
tune:
|
|
46
|
+
max-num-seqs:
|
|
47
|
+
values: [8, 16]
|
|
48
|
+
benchmark:
|
|
49
|
+
runs:
|
|
50
|
+
- name: throughput
|
|
51
|
+
profile:
|
|
52
|
+
kind: throughput
|
|
53
|
+
max_concurrency: 16
|
|
54
|
+
constraints:
|
|
55
|
+
- kind: max_requests
|
|
56
|
+
count: 10
|
|
57
|
+
data:
|
|
58
|
+
- kind: synthetic_text
|
|
59
|
+
prompt_tokens: 32
|
|
60
|
+
output_tokens: 16
|
|
61
|
+
optimization:
|
|
62
|
+
maximize: output_tokens_per_second
|
|
63
|
+
sampler: tpe
|
|
64
|
+
trials: 2
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Run it:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
vtune --config experiment.yaml
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The short form is `vtune -c experiment.yaml`. The command validates the file,
|
|
74
|
+
runs the experiment, persists results, and generates its exports and report.
|
|
75
|
+
vTune binds vLLM to `127.0.0.1` by default. Set `server.args.host` explicitly
|
|
76
|
+
only when the benchmark server must be reachable from another host.
|
|
77
|
+
|
|
78
|
+
Terminal output is concise by default. To stream vLLM and GuideLLM logs:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
vtune --config experiment.yaml --verbose
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The persistent equivalent uses GuideLLM's logging level names:
|
|
85
|
+
|
|
86
|
+
```yaml
|
|
87
|
+
logging:
|
|
88
|
+
level: DEBUG
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Supported levels are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.
|
|
92
|
+
Full per-trial log files are always saved. `--verbose` overrides the configured
|
|
93
|
+
level with `DEBUG` for that invocation.
|
|
94
|
+
|
|
95
|
+
Retry one or more selected trials into a new immutable linked run:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
vtune retry --run runs/EXPERIMENT/RUN_ID \
|
|
99
|
+
--trial trial-0001 --trial trial-0004
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The source run is never modified.
|
|
103
|
+
|
|
104
|
+
Display every stored vLLM and GuideLLM command for a trial without executing
|
|
105
|
+
anything:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
vtune reproduce --run runs/EXPERIMENT/RUN_ID --trial trial-0001
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Each completed run also contains a self-contained `report.html` decision
|
|
112
|
+
dashboard with the best observed configuration, baseline comparison, score
|
|
113
|
+
history, throughput/latency tradeoff, and observed parameter effects.
|
|
114
|
+
|
|
115
|
+
Random and TPE runs never execute the same resolved configuration twice.
|
|
116
|
+
`optimization.trials` cannot exceed the number of unique configurations in
|
|
117
|
+
the declared search space.
|
|
118
|
+
|
|
119
|
+
## Product documents
|
|
120
|
+
|
|
121
|
+
- [First MVP specification](docs/MVP_SPEC.md)
|
|
122
|
+
- [Future implementation roadmap](docs/ROADMAP.md)
|
|
123
|
+
- [Architecture overview and early sketch](docs/ARCHITECTURE.md)
|
|
124
|
+
- [Editable Draw.io architecture diagram](docs/vtune-architecture.drawio)
|
|
125
|
+
- [Contributor guide](CONTRIBUTING.md)
|
|
126
|
+
- [Release notes](CHANGELOG.md)
|
|
127
|
+
|
|
128
|
+
The MVP specification defines the first releasable version and its acceptance
|
|
129
|
+
criteria. The roadmap describes capabilities that should be designed for now
|
|
130
|
+
but implemented after the core experiment loop is reliable.
|
|
131
|
+
|
|
132
|
+
vTune is available under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "vtune"
|
|
7
|
+
version = "0.1.0a1"
|
|
8
|
+
description = "Local-first experimentation for vLLM serving configurations"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
keywords = ["vllm", "guidellm", "benchmark", "optimization", "inference"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Operating System :: POSIX :: Linux",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Typing :: Typed",
|
|
22
|
+
]
|
|
23
|
+
dependencies = ["Optuna>=4.0,<5", "PyYAML>=6.0"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Repository = "https://github.com/brtydse100/vTune"
|
|
27
|
+
Issues = "https://github.com/brtydse100/vTune/issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
vtune = "vtune.cli:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
vtune-0.1.0a1/setup.cfg
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""GuideLLM configuration, command construction, and result normalization."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from vtune.config.models import VTuneConfig
|
|
12
|
+
from vtune.domain.benchmark import BenchmarkResult, WorkloadResult
|
|
13
|
+
from vtune.benchmarks.timing import normalize_durations
|
|
14
|
+
|
|
15
|
+
_RUN_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class GuideLLMPlan:
|
|
20
|
+
run_name: str
|
|
21
|
+
argv: tuple[str, ...]
|
|
22
|
+
directory: Path
|
|
23
|
+
json_path: Path
|
|
24
|
+
log_path: Path
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def configured_runs(config: VTuneConfig) -> tuple[Mapping[str, object], ...]:
|
|
28
|
+
unknown = set(config.benchmark) - {"runs", "repeats"}
|
|
29
|
+
if unknown:
|
|
30
|
+
raise ValueError(f"Unsupported benchmark setting(s): {', '.join(sorted(unknown))}")
|
|
31
|
+
runs = config.benchmark.get("runs")
|
|
32
|
+
if not isinstance(runs, list) or not runs:
|
|
33
|
+
raise ValueError("'benchmark.runs' must be a non-empty list")
|
|
34
|
+
validated: list[Mapping[str, object]] = []
|
|
35
|
+
names: set[str] = set()
|
|
36
|
+
for index, value in enumerate(runs):
|
|
37
|
+
run = _mapping(value, f"run {index}")
|
|
38
|
+
unknown_run = set(run) - {
|
|
39
|
+
"name", "request_format", "profile", "constraints", "data"
|
|
40
|
+
}
|
|
41
|
+
if unknown_run:
|
|
42
|
+
options = ", ".join(sorted(unknown_run))
|
|
43
|
+
raise ValueError(f"Unsupported setting(s) in benchmark run {index}: {options}")
|
|
44
|
+
name = run.get("name")
|
|
45
|
+
if not isinstance(name, str) or not _RUN_NAME.fullmatch(name):
|
|
46
|
+
raise ValueError("benchmark run names must use letters, numbers, '_' or '-'")
|
|
47
|
+
if name in names:
|
|
48
|
+
raise ValueError(f"duplicate benchmark run name: {name}")
|
|
49
|
+
names.add(name)
|
|
50
|
+
data = run.get("data")
|
|
51
|
+
if not isinstance(data, list) or len(data) != 1:
|
|
52
|
+
raise ValueError(f"benchmark run '{name}' must configure exactly one dataset")
|
|
53
|
+
validated.append(run)
|
|
54
|
+
return tuple(validated)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def configured_repeats(config: VTuneConfig) -> int:
|
|
58
|
+
value = config.benchmark.get("repeats", 1)
|
|
59
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
60
|
+
raise ValueError("benchmark.repeats must be a positive integer")
|
|
61
|
+
return value
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def build_plan(
|
|
65
|
+
config: VTuneConfig, run: Mapping[str, object], endpoint: str, artifacts: Path,
|
|
66
|
+
) -> GuideLLMPlan:
|
|
67
|
+
name = run["name"]
|
|
68
|
+
request_format = run.get("request_format", "/v1/completions")
|
|
69
|
+
if not isinstance(request_format, str) or not request_format.strip():
|
|
70
|
+
raise ValueError("benchmark request_format must be a non-empty string")
|
|
71
|
+
directory = Path(artifacts) / str(name)
|
|
72
|
+
json_path = directory / "results.json"
|
|
73
|
+
argv = [
|
|
74
|
+
"guidellm", "run", "--backend",
|
|
75
|
+
_serialize({"kind": "openai_http", "target": endpoint,
|
|
76
|
+
"model": config.model.path, "request_format": request_format}),
|
|
77
|
+
"--profile", _serialize(_mapping(run.get("profile"), "profile")),
|
|
78
|
+
]
|
|
79
|
+
for option, label in (("constraints", "constraint"), ("data", "data")):
|
|
80
|
+
values = run.get(option, [])
|
|
81
|
+
if not isinstance(values, list):
|
|
82
|
+
raise ValueError(f"benchmark run '{option}' must be a list")
|
|
83
|
+
for value in values:
|
|
84
|
+
if label == "constraint" and isinstance(value, dict) and value.get("kind") == "max_duration":
|
|
85
|
+
value = {**value, "seconds": normalize_durations(value.get("seconds"))}
|
|
86
|
+
argv.extend((f"--{label}", _serialize(_mapping(value, label))))
|
|
87
|
+
argv.extend(("--output", f"kind=json,path={json_path}",
|
|
88
|
+
"--disable-console-interactive"))
|
|
89
|
+
return GuideLLMPlan(str(name), tuple(argv), directory, json_path,
|
|
90
|
+
directory / "benchmark.log")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def parse_result(path: Path, run_name: str) -> BenchmarkResult:
|
|
94
|
+
source = Path(path)
|
|
95
|
+
try:
|
|
96
|
+
root = _mapping(json.loads(source.read_text(encoding="utf-8")), "result")
|
|
97
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
|
98
|
+
raise ValueError(f"Cannot read GuideLLM JSON result: {error}") from error
|
|
99
|
+
metadata = _mapping(root.get("metadata"), "metadata")
|
|
100
|
+
version = metadata.get("guidellm_version")
|
|
101
|
+
if not isinstance(version, str) or not version.strip():
|
|
102
|
+
raise ValueError("GuideLLM result is missing its version")
|
|
103
|
+
benchmarks = root.get("benchmarks")
|
|
104
|
+
if not isinstance(benchmarks, list) or not benchmarks:
|
|
105
|
+
raise ValueError("GuideLLM result must contain benchmarks")
|
|
106
|
+
workloads = tuple(
|
|
107
|
+
WorkloadResult(index, _mapping(item.get("config"), f"benchmark {index} config"),
|
|
108
|
+
_mapping(item.get("metrics"), f"benchmark {index} metrics"))
|
|
109
|
+
for index, value in enumerate(benchmarks)
|
|
110
|
+
for item in (_mapping(value, f"benchmark {index}"),)
|
|
111
|
+
)
|
|
112
|
+
return BenchmarkResult(run_name, "guidellm", version, workloads, source)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _serialize(values: Mapping[str, object]) -> str:
|
|
116
|
+
kind = values.get("kind")
|
|
117
|
+
if not isinstance(kind, str) or not kind.strip():
|
|
118
|
+
raise ValueError("GuideLLM options require a non-empty 'kind'")
|
|
119
|
+
return ",".join(pair for key, value in values.items() for pair in _pairs(key, value))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _pairs(path: str, value: object) -> tuple[str, ...]:
|
|
123
|
+
if isinstance(value, Mapping):
|
|
124
|
+
return tuple(pair for key, item in value.items()
|
|
125
|
+
for pair in _pairs(f"{path}.{key}", item))
|
|
126
|
+
if isinstance(value, list | tuple):
|
|
127
|
+
return tuple(pair for index, item in enumerate(value)
|
|
128
|
+
for pair in _pairs(f"{path}[{index}]", item))
|
|
129
|
+
if isinstance(value, bool):
|
|
130
|
+
rendered = str(value).lower()
|
|
131
|
+
if isinstance(value, str | int | float) and not isinstance(value, bool):
|
|
132
|
+
rendered = str(value)
|
|
133
|
+
elif value is None:
|
|
134
|
+
rendered = "null"
|
|
135
|
+
elif not isinstance(value, bool):
|
|
136
|
+
raise ValueError(f"Unsupported GuideLLM value at '{path}'")
|
|
137
|
+
if "," in rendered:
|
|
138
|
+
raise ValueError(f"GuideLLM string values cannot contain commas: '{path}'")
|
|
139
|
+
return (f"{path}={rendered}",)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _mapping(value: object, label: str) -> Mapping[str, object]:
|
|
143
|
+
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
|
144
|
+
raise ValueError(f"GuideLLM {label} must be an object")
|
|
145
|
+
return value
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Duration parsing and GuideLLM timeout estimation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
_DURATION = re.compile(r"^(\d+(?:\.\d+)?)\s*([smh]?)$")
|
|
9
|
+
_UNITS = {"": 1, "s": 1, "m": 60, "h": 3600}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_duration(value: object, label: str = "duration") -> float:
|
|
13
|
+
if isinstance(value, bool):
|
|
14
|
+
raise ValueError(f"'{label}' must be a positive duration")
|
|
15
|
+
if isinstance(value, int | float):
|
|
16
|
+
seconds = float(value)
|
|
17
|
+
elif isinstance(value, str) and (match := _DURATION.fullmatch(value.strip().lower())):
|
|
18
|
+
seconds = float(match.group(1)) * _UNITS[match.group(2)]
|
|
19
|
+
else:
|
|
20
|
+
raise ValueError(f"'{label}' must use seconds, '30s', '2m', or '1h'")
|
|
21
|
+
if seconds <= 0:
|
|
22
|
+
raise ValueError(f"'{label}' must be positive")
|
|
23
|
+
return seconds
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def timeout_for_run(run: Mapping[str, object], configured: object = "auto") -> float:
|
|
27
|
+
"""Return an explicit timeout or estimate duration plus a safety margin."""
|
|
28
|
+
if configured != "auto":
|
|
29
|
+
return parse_duration(configured, "timeouts.benchmark")
|
|
30
|
+
duration = _duration_constraint(run)
|
|
31
|
+
if duration is None:
|
|
32
|
+
return 180.0
|
|
33
|
+
expected = duration * _strategy_count(run.get("profile"))
|
|
34
|
+
return expected + max(30.0, expected * 0.25)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def normalize_durations(value: object) -> object:
|
|
38
|
+
"""Convert duration strings recursively before passing values to GuideLLM."""
|
|
39
|
+
if isinstance(value, list):
|
|
40
|
+
return [normalize_durations(item) for item in value]
|
|
41
|
+
if isinstance(value, str):
|
|
42
|
+
return parse_duration(value)
|
|
43
|
+
return value
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _duration_constraint(run: Mapping[str, object]) -> float | None:
|
|
47
|
+
constraints = run.get("constraints", [])
|
|
48
|
+
if not isinstance(constraints, list):
|
|
49
|
+
return None
|
|
50
|
+
for constraint in constraints:
|
|
51
|
+
if isinstance(constraint, Mapping) and constraint.get("kind") == "max_duration":
|
|
52
|
+
seconds = constraint.get("seconds")
|
|
53
|
+
if isinstance(seconds, list):
|
|
54
|
+
return sum(parse_duration(item, "max_duration.seconds") for item in seconds)
|
|
55
|
+
return parse_duration(seconds, "max_duration.seconds")
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _strategy_count(profile: object) -> int:
|
|
60
|
+
if not isinstance(profile, Mapping):
|
|
61
|
+
return 1
|
|
62
|
+
for key in ("streams", "rates"):
|
|
63
|
+
value = profile.get(key)
|
|
64
|
+
if isinstance(value, list):
|
|
65
|
+
return max(1, len(value))
|
|
66
|
+
size = profile.get("sweep_size")
|
|
67
|
+
return size if isinstance(size, int) and size > 0 else 1
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Command-line entry point for vTune."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import asyncio
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from vtune.config.errors import ConfigError
|
|
12
|
+
from vtune.config.loader import load_config
|
|
13
|
+
from vtune.cli_options import CLIUsageError, validate_cli_options
|
|
14
|
+
from vtune.lifecycle import load_retry_plan
|
|
15
|
+
from vtune.orchestrator import Orchestrator
|
|
16
|
+
from vtune.reproduction.display import reproduce_trial
|
|
17
|
+
from vtune.reproduction.export import export_vllm_command
|
|
18
|
+
from vtune.terminal import with_debug_logging
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
22
|
+
parser = argparse.ArgumentParser(
|
|
23
|
+
prog="vtune", description="Experiment with vLLM serving configurations."
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"action", nargs="?", choices=("validate", "export", "reproduce", "retry"),
|
|
27
|
+
help="Post-run action; omit to start a new experiment.")
|
|
28
|
+
parser.add_argument("--config", "-c", metavar="YAML",
|
|
29
|
+
help="Experiment YAML for a new run or validation.")
|
|
30
|
+
parser.add_argument("--run", type=Path, metavar="DIRECTORY",
|
|
31
|
+
help="Existing immutable run directory.")
|
|
32
|
+
parser.add_argument("--trial", action="append", metavar="ID",
|
|
33
|
+
help="Trial ID; repeat the option when retrying several trials.")
|
|
34
|
+
parser.add_argument("--verbose", action="store_true",
|
|
35
|
+
help="Override logging.level with DEBUG and stream child logs.")
|
|
36
|
+
return parser
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
40
|
+
args = build_parser().parse_args(argv)
|
|
41
|
+
try:
|
|
42
|
+
validate_cli_options(args)
|
|
43
|
+
if args.action == "export":
|
|
44
|
+
print(export_vllm_command(args.run, args.trial[0]))
|
|
45
|
+
return 0
|
|
46
|
+
if args.action == "reproduce":
|
|
47
|
+
print(reproduce_trial(args.run, args.trial[0]))
|
|
48
|
+
return 0
|
|
49
|
+
if args.action == "retry":
|
|
50
|
+
plan = load_retry_plan(args.run, args.trial)
|
|
51
|
+
for warning in plan.warnings:
|
|
52
|
+
print(f"Integrity warning: {warning}", file=sys.stderr)
|
|
53
|
+
retry_config = with_debug_logging(plan.config) if args.verbose else plan.config
|
|
54
|
+
outcome = asyncio.run(Orchestrator(
|
|
55
|
+
retry_config, plan.trials, plan.source_run_id, plan.sources,
|
|
56
|
+
).run())
|
|
57
|
+
else:
|
|
58
|
+
config = load_config(args.config)
|
|
59
|
+
config = with_debug_logging(config) if args.verbose else config
|
|
60
|
+
if args.action == "validate":
|
|
61
|
+
Orchestrator(config).validate()
|
|
62
|
+
print(f"Configuration valid: {config.experiment.name}")
|
|
63
|
+
print(f"Model: {config.model.path}")
|
|
64
|
+
return 0
|
|
65
|
+
outcome = asyncio.run(Orchestrator(config).run())
|
|
66
|
+
except CLIUsageError as error:
|
|
67
|
+
print(f"Command error: {error}", file=sys.stderr)
|
|
68
|
+
return 2
|
|
69
|
+
except ConfigError as error:
|
|
70
|
+
print(f"Configuration error: {error}", file=sys.stderr)
|
|
71
|
+
return 2
|
|
72
|
+
except (OSError, TypeError, ValueError) as error:
|
|
73
|
+
print(f"Experiment error: {error}", file=sys.stderr)
|
|
74
|
+
return 1
|
|
75
|
+
except KeyboardInterrupt:
|
|
76
|
+
print("Experiment interrupted", file=sys.stderr)
|
|
77
|
+
return 130
|
|
78
|
+
print(outcome.summary)
|
|
79
|
+
if outcome.status == "interrupted":
|
|
80
|
+
return 130
|
|
81
|
+
completed = any(report.status.value == "completed" for report in outcome.trials)
|
|
82
|
+
return 0 if completed else 1
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
raise SystemExit(main())
|