vtune 0.1.0a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. vtune-0.1.0a1/LICENSE +21 -0
  2. vtune-0.1.0a1/PKG-INFO +154 -0
  3. vtune-0.1.0a1/README.md +132 -0
  4. vtune-0.1.0a1/pyproject.toml +33 -0
  5. vtune-0.1.0a1/setup.cfg +4 -0
  6. vtune-0.1.0a1/src/vtune/__init__.py +7 -0
  7. vtune-0.1.0a1/src/vtune/benchmarks/__init__.py +9 -0
  8. vtune-0.1.0a1/src/vtune/benchmarks/guidellm.py +145 -0
  9. vtune-0.1.0a1/src/vtune/benchmarks/timing.py +67 -0
  10. vtune-0.1.0a1/src/vtune/cli.py +86 -0
  11. vtune-0.1.0a1/src/vtune/cli_options.py +38 -0
  12. vtune-0.1.0a1/src/vtune/config/__init__.py +6 -0
  13. vtune-0.1.0a1/src/vtune/config/errors.py +17 -0
  14. vtune-0.1.0a1/src/vtune/config/loader.py +147 -0
  15. vtune-0.1.0a1/src/vtune/config/models.py +42 -0
  16. vtune-0.1.0a1/src/vtune/config/runtime.py +56 -0
  17. vtune-0.1.0a1/src/vtune/domain/__init__.py +25 -0
  18. vtune-0.1.0a1/src/vtune/domain/attempt_report.py +18 -0
  19. vtune-0.1.0a1/src/vtune/domain/benchmark.py +46 -0
  20. vtune-0.1.0a1/src/vtune/domain/models.py +69 -0
  21. vtune-0.1.0a1/src/vtune/domain/results.py +59 -0
  22. vtune-0.1.0a1/src/vtune/domain/states.py +31 -0
  23. vtune-0.1.0a1/src/vtune/domain/trial_report.py +70 -0
  24. vtune-0.1.0a1/src/vtune/lifecycle/__init__.py +5 -0
  25. vtune-0.1.0a1/src/vtune/lifecycle/integrity.py +102 -0
  26. vtune-0.1.0a1/src/vtune/lifecycle/retry.py +106 -0
  27. vtune-0.1.0a1/src/vtune/managers/__init__.py +9 -0
  28. vtune-0.1.0a1/src/vtune/managers/results.py +65 -0
  29. vtune-0.1.0a1/src/vtune/managers/run_results.py +114 -0
  30. vtune-0.1.0a1/src/vtune/managers/run_session.py +69 -0
  31. vtune-0.1.0a1/src/vtune/managers/scoring.py +57 -0
  32. vtune-0.1.0a1/src/vtune/managers/trial.py +97 -0
  33. vtune-0.1.0a1/src/vtune/orchestrator.py +150 -0
  34. vtune-0.1.0a1/src/vtune/py.typed +1 -0
  35. vtune-0.1.0a1/src/vtune/reporting/__init__.py +5 -0
  36. vtune-0.1.0a1/src/vtune/reporting/analysis.py +80 -0
  37. vtune-0.1.0a1/src/vtune/reporting/charts.py +94 -0
  38. vtune-0.1.0a1/src/vtune/reporting/context.py +17 -0
  39. vtune-0.1.0a1/src/vtune/reporting/dashboard.py +113 -0
  40. vtune-0.1.0a1/src/vtune/reporting/reporter.py +57 -0
  41. vtune-0.1.0a1/src/vtune/reporting/tables.py +91 -0
  42. vtune-0.1.0a1/src/vtune/reproduction/__init__.py +1 -0
  43. vtune-0.1.0a1/src/vtune/reproduction/display.py +63 -0
  44. vtune-0.1.0a1/src/vtune/reproduction/export.py +13 -0
  45. vtune-0.1.0a1/src/vtune/reproduction/manifest.py +72 -0
  46. vtune-0.1.0a1/src/vtune/reproduction/metadata.py +65 -0
  47. vtune-0.1.0a1/src/vtune/reproduction/models.py +36 -0
  48. vtune-0.1.0a1/src/vtune/reproduction/reader.py +48 -0
  49. vtune-0.1.0a1/src/vtune/reproduction/redaction.py +39 -0
  50. vtune-0.1.0a1/src/vtune/search/__init__.py +6 -0
  51. vtune-0.1.0a1/src/vtune/search/factory.py +40 -0
  52. vtune-0.1.0a1/src/vtune/search/fixed_session.py +26 -0
  53. vtune-0.1.0a1/src/vtune/search/grid.py +65 -0
  54. vtune-0.1.0a1/src/vtune/search/grid_session.py +29 -0
  55. vtune-0.1.0a1/src/vtune/search/optuna_session.py +121 -0
  56. vtune-0.1.0a1/src/vtune/search/strategy.py +18 -0
  57. vtune-0.1.0a1/src/vtune/terminal.py +30 -0
  58. vtune-0.1.0a1/src/vtune/workers/__init__.py +21 -0
  59. vtune-0.1.0a1/src/vtune/workers/attempts.py +15 -0
  60. vtune-0.1.0a1/src/vtune/workers/base.py +32 -0
  61. vtune-0.1.0a1/src/vtune/workers/benchmark.py +98 -0
  62. vtune-0.1.0a1/src/vtune/workers/configuration.py +105 -0
  63. vtune-0.1.0a1/src/vtune/workers/factory.py +43 -0
  64. vtune-0.1.0a1/src/vtune/workers/failure_details.py +40 -0
  65. vtune-0.1.0a1/src/vtune/workers/process.py +147 -0
  66. vtune-0.1.0a1/src/vtune/workers/readiness.py +116 -0
  67. vtune-0.1.0a1/src/vtune/workers/vllm.py +67 -0
  68. vtune-0.1.0a1/src/vtune.egg-info/PKG-INFO +154 -0
  69. vtune-0.1.0a1/src/vtune.egg-info/SOURCES.txt +71 -0
  70. vtune-0.1.0a1/src/vtune.egg-info/dependency_links.txt +1 -0
  71. vtune-0.1.0a1/src/vtune.egg-info/entry_points.txt +2 -0
  72. vtune-0.1.0a1/src/vtune.egg-info/requires.txt +2 -0
  73. vtune-0.1.0a1/src/vtune.egg-info/top_level.txt +1 -0
vtune-0.1.0a1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 vTune contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
vtune-0.1.0a1/PKG-INFO ADDED
@@ -0,0 +1,154 @@
1
+ Metadata-Version: 2.4
2
+ Name: vtune
3
+ Version: 0.1.0a1
4
+ Summary: Local-first experimentation for vLLM serving configurations
5
+ License-Expression: MIT
6
+ Project-URL: Repository, https://github.com/brtydse100/vTune
7
+ Project-URL: Issues, https://github.com/brtydse100/vTune/issues
8
+ Keywords: vllm,guidellm,benchmark,optimization,inference
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Environment :: Console
11
+ Classifier: Operating System :: POSIX :: Linux
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Typing :: Typed
16
+ Requires-Python: >=3.11
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: Optuna<5,>=4.0
20
+ Requires-Dist: PyYAML>=6.0
21
+ Dynamic: license-file
22
+
23
+ # vTune
24
+
25
+ vTune is a local-first experimentation and optimization tool for vLLM serving
26
+ configurations. Users define the parameters and workloads they care about;
27
+ vTune manages the server lifecycle, runs repeatable benchmarks, explores the
28
+ search space, and reports which configurations performed best.
29
+
30
+ vTune is alpha software targeting Linux with NVIDIA GPUs and Python 3.11–3.12.
31
+
32
+ Tested integrations:
33
+
34
+ - vLLM 0.10.2 with GuideLLM 0.7.3 on WSL2 and an RTX 3080.
35
+ - vLLM 0.28.0 with GuideLLM 0.7.3 on the same system. WSL2 required
36
+ `VLLM_USE_V2_MODEL_RUNNER: "0"` because UVA was unavailable and
37
+ `VLLM_USE_FLASHINFER_SAMPLER: "0"` because the CUDA compiler toolkit was
38
+ not installed. Native Linux systems may not require these settings.
39
+
40
+ Other combinations may work but are not yet verified.
41
+
42
+ The published `py3-none-any` wheel installs on Linux and Windows. Configuration
43
+ validation and stored-result inspection work on Windows, but starting an
44
+ experiment is supported only on Linux because vLLM has no native Windows
45
+ runtime.
46
+
47
+ ## Quick start
48
+
49
+ Requirements: Linux, an NVIDIA GPU, a local model directory, and working
50
+ `vllm` and `guidellm` commands. Install this checkout with:
51
+
52
+ ```bash
53
+ pip install -e .
54
+ ```
55
+
56
+ Create `experiment.yaml`:
57
+
58
+ ```yaml
59
+ schema_version: 1
60
+ experiment:
61
+ name: first-run
62
+ model:
63
+ path: /models/opt-125m
64
+ server:
65
+ args:
66
+ gpu-memory-utilization: 0.8
67
+ tune:
68
+ max-num-seqs:
69
+ values: [8, 16]
70
+ benchmark:
71
+ runs:
72
+ - name: throughput
73
+ profile:
74
+ kind: throughput
75
+ max_concurrency: 16
76
+ constraints:
77
+ - kind: max_requests
78
+ count: 10
79
+ data:
80
+ - kind: synthetic_text
81
+ prompt_tokens: 32
82
+ output_tokens: 16
83
+ optimization:
84
+ maximize: output_tokens_per_second
85
+ sampler: tpe
86
+ trials: 2
87
+ ```
88
+
89
+ Run it:
90
+
91
+ ```bash
92
+ vtune --config experiment.yaml
93
+ ```
94
+
95
+ The short form is `vtune -c experiment.yaml`. The command validates the file,
96
+ runs the experiment, persists results, and generates its exports and report.
97
+ vTune binds vLLM to `127.0.0.1` by default. Set `server.args.host` explicitly
98
+ only when the benchmark server must be reachable from another host.
99
+
100
+ Terminal output is concise by default. To stream vLLM and GuideLLM logs:
101
+
102
+ ```bash
103
+ vtune --config experiment.yaml --verbose
104
+ ```
105
+
106
+ The persistent equivalent uses GuideLLM's logging level names:
107
+
108
+ ```yaml
109
+ logging:
110
+ level: DEBUG
111
+ ```
112
+
113
+ Supported levels are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.
114
+ Full per-trial log files are always saved. `--verbose` overrides the configured
115
+ level with `DEBUG` for that invocation.
116
+
117
+ Retry one or more selected trials into a new immutable linked run:
118
+
119
+ ```bash
120
+ vtune retry --run runs/EXPERIMENT/RUN_ID \
121
+ --trial trial-0001 --trial trial-0004
122
+ ```
123
+
124
+ The source run is never modified.
125
+
126
+ Display every stored vLLM and GuideLLM command for a trial without executing
127
+ anything:
128
+
129
+ ```bash
130
+ vtune reproduce --run runs/EXPERIMENT/RUN_ID --trial trial-0001
131
+ ```
132
+
133
+ Each completed run also contains a self-contained `report.html` decision
134
+ dashboard with the best observed configuration, baseline comparison, score
135
+ history, throughput/latency tradeoff, and observed parameter effects.
136
+
137
+ Random and TPE runs never execute the same resolved configuration twice.
138
+ `optimization.trials` cannot exceed the number of unique configurations in
139
+ the declared search space.
140
+
141
+ ## Product documents
142
+
143
+ - [First MVP specification](docs/MVP_SPEC.md)
144
+ - [Future implementation roadmap](docs/ROADMAP.md)
145
+ - [Architecture overview and early sketch](docs/ARCHITECTURE.md)
146
+ - [Editable Draw.io architecture diagram](docs/vtune-architecture.drawio)
147
+ - [Contributor guide](CONTRIBUTING.md)
148
+ - [Release notes](CHANGELOG.md)
149
+
150
+ The MVP specification defines the first releasable version and its acceptance
151
+ criteria. The roadmap describes capabilities that should be designed for now
152
+ but implemented after the core experiment loop is reliable.
153
+
154
+ vTune is available under the [MIT License](LICENSE).
@@ -0,0 +1,132 @@
1
+ # vTune
2
+
3
+ vTune is a local-first experimentation and optimization tool for vLLM serving
4
+ configurations. Users define the parameters and workloads they care about;
5
+ vTune manages the server lifecycle, runs repeatable benchmarks, explores the
6
+ search space, and reports which configurations performed best.
7
+
8
+ vTune is alpha software targeting Linux with NVIDIA GPUs and Python 3.11–3.12.
9
+
10
+ Tested integrations:
11
+
12
+ - vLLM 0.10.2 with GuideLLM 0.7.3 on WSL2 and an RTX 3080.
13
+ - vLLM 0.28.0 with GuideLLM 0.7.3 on the same system. WSL2 required
14
+ `VLLM_USE_V2_MODEL_RUNNER: "0"` because UVA was unavailable and
15
+ `VLLM_USE_FLASHINFER_SAMPLER: "0"` because the CUDA compiler toolkit was
16
+ not installed. Native Linux systems may not require these settings.
17
+
18
+ Other combinations may work but are not yet verified.
19
+
20
+ The published `py3-none-any` wheel installs on Linux and Windows. Configuration
21
+ validation and stored-result inspection work on Windows, but starting an
22
+ experiment is supported only on Linux because vLLM has no native Windows
23
+ runtime.
24
+
25
+ ## Quick start
26
+
27
+ Requirements: Linux, an NVIDIA GPU, a local model directory, and working
28
+ `vllm` and `guidellm` commands. Install this checkout with:
29
+
30
+ ```bash
31
+ pip install -e .
32
+ ```
33
+
34
+ Create `experiment.yaml`:
35
+
36
+ ```yaml
37
+ schema_version: 1
38
+ experiment:
39
+ name: first-run
40
+ model:
41
+ path: /models/opt-125m
42
+ server:
43
+ args:
44
+ gpu-memory-utilization: 0.8
45
+ tune:
46
+ max-num-seqs:
47
+ values: [8, 16]
48
+ benchmark:
49
+ runs:
50
+ - name: throughput
51
+ profile:
52
+ kind: throughput
53
+ max_concurrency: 16
54
+ constraints:
55
+ - kind: max_requests
56
+ count: 10
57
+ data:
58
+ - kind: synthetic_text
59
+ prompt_tokens: 32
60
+ output_tokens: 16
61
+ optimization:
62
+ maximize: output_tokens_per_second
63
+ sampler: tpe
64
+ trials: 2
65
+ ```
66
+
67
+ Run it:
68
+
69
+ ```bash
70
+ vtune --config experiment.yaml
71
+ ```
72
+
73
+ The short form is `vtune -c experiment.yaml`. The command validates the file,
74
+ runs the experiment, persists results, and generates its exports and report.
75
+ vTune binds vLLM to `127.0.0.1` by default. Set `server.args.host` explicitly
76
+ only when the benchmark server must be reachable from another host.
77
+
78
+ Terminal output is concise by default. To stream vLLM and GuideLLM logs:
79
+
80
+ ```bash
81
+ vtune --config experiment.yaml --verbose
82
+ ```
83
+
84
+ The persistent equivalent uses GuideLLM's logging level names:
85
+
86
+ ```yaml
87
+ logging:
88
+ level: DEBUG
89
+ ```
90
+
91
+ Supported levels are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.
92
+ Full per-trial log files are always saved. `--verbose` overrides the configured
93
+ level with `DEBUG` for that invocation.
94
+
95
+ Retry one or more selected trials into a new immutable linked run:
96
+
97
+ ```bash
98
+ vtune retry --run runs/EXPERIMENT/RUN_ID \
99
+ --trial trial-0001 --trial trial-0004
100
+ ```
101
+
102
+ The source run is never modified.
103
+
104
+ Display every stored vLLM and GuideLLM command for a trial without executing
105
+ anything:
106
+
107
+ ```bash
108
+ vtune reproduce --run runs/EXPERIMENT/RUN_ID --trial trial-0001
109
+ ```
110
+
111
+ Each completed run also contains a self-contained `report.html` decision
112
+ dashboard with the best observed configuration, baseline comparison, score
113
+ history, throughput/latency tradeoff, and observed parameter effects.
114
+
115
+ Random and TPE runs never execute the same resolved configuration twice.
116
+ `optimization.trials` cannot exceed the number of unique configurations in
117
+ the declared search space.
118
+
119
+ ## Product documents
120
+
121
+ - [First MVP specification](docs/MVP_SPEC.md)
122
+ - [Future implementation roadmap](docs/ROADMAP.md)
123
+ - [Architecture overview and early sketch](docs/ARCHITECTURE.md)
124
+ - [Editable Draw.io architecture diagram](docs/vtune-architecture.drawio)
125
+ - [Contributor guide](CONTRIBUTING.md)
126
+ - [Release notes](CHANGELOG.md)
127
+
128
+ The MVP specification defines the first releasable version and its acceptance
129
+ criteria. The roadmap describes capabilities that should be designed for now
130
+ but implemented after the core experiment loop is reliable.
131
+
132
+ vTune is available under the [MIT License](LICENSE).
@@ -0,0 +1,33 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "vtune"
7
+ version = "0.1.0a1"
8
+ description = "Local-first experimentation for vLLM serving configurations"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ keywords = ["vllm", "guidellm", "benchmark", "optimization", "inference"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Environment :: Console",
17
+ "Operating System :: POSIX :: Linux",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3.11",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Typing :: Typed",
22
+ ]
23
+ dependencies = ["Optuna>=4.0,<5", "PyYAML>=6.0"]
24
+
25
+ [project.urls]
26
+ Repository = "https://github.com/brtydse100/vTune"
27
+ Issues = "https://github.com/brtydse100/vTune/issues"
28
+
29
+ [project.scripts]
30
+ vtune = "vtune.cli:main"
31
+
32
+ [tool.setuptools.packages.find]
33
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,7 @@
1
+ """vTune package."""
2
+
3
+ __version__ = "0.1.0a1"
4
+
5
+ from .orchestrator import Orchestrator, RunOutcome
6
+
7
+ __all__ = ["Orchestrator", "RunOutcome"]
@@ -0,0 +1,9 @@
1
+ """Benchmark backend adapters."""
2
+
3
+ from .guidellm import (
4
+ GuideLLMPlan, build_plan, configured_repeats, configured_runs, parse_result,
5
+ )
6
+
7
+ __all__ = [
8
+ "GuideLLMPlan", "build_plan", "configured_repeats", "configured_runs", "parse_result"
9
+ ]
@@ -0,0 +1,145 @@
1
+ """GuideLLM configuration, command construction, and result normalization."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ from dataclasses import dataclass
7
+ import json
8
+ from pathlib import Path
9
+ import re
10
+
11
+ from vtune.config.models import VTuneConfig
12
+ from vtune.domain.benchmark import BenchmarkResult, WorkloadResult
13
+ from vtune.benchmarks.timing import normalize_durations
14
+
15
+ _RUN_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$")
16
+
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class GuideLLMPlan:
20
+ run_name: str
21
+ argv: tuple[str, ...]
22
+ directory: Path
23
+ json_path: Path
24
+ log_path: Path
25
+
26
+
27
+ def configured_runs(config: VTuneConfig) -> tuple[Mapping[str, object], ...]:
28
+ unknown = set(config.benchmark) - {"runs", "repeats"}
29
+ if unknown:
30
+ raise ValueError(f"Unsupported benchmark setting(s): {', '.join(sorted(unknown))}")
31
+ runs = config.benchmark.get("runs")
32
+ if not isinstance(runs, list) or not runs:
33
+ raise ValueError("'benchmark.runs' must be a non-empty list")
34
+ validated: list[Mapping[str, object]] = []
35
+ names: set[str] = set()
36
+ for index, value in enumerate(runs):
37
+ run = _mapping(value, f"run {index}")
38
+ unknown_run = set(run) - {
39
+ "name", "request_format", "profile", "constraints", "data"
40
+ }
41
+ if unknown_run:
42
+ options = ", ".join(sorted(unknown_run))
43
+ raise ValueError(f"Unsupported setting(s) in benchmark run {index}: {options}")
44
+ name = run.get("name")
45
+ if not isinstance(name, str) or not _RUN_NAME.fullmatch(name):
46
+ raise ValueError("benchmark run names must use letters, numbers, '_' or '-'")
47
+ if name in names:
48
+ raise ValueError(f"duplicate benchmark run name: {name}")
49
+ names.add(name)
50
+ data = run.get("data")
51
+ if not isinstance(data, list) or len(data) != 1:
52
+ raise ValueError(f"benchmark run '{name}' must configure exactly one dataset")
53
+ validated.append(run)
54
+ return tuple(validated)
55
+
56
+
57
+ def configured_repeats(config: VTuneConfig) -> int:
58
+ value = config.benchmark.get("repeats", 1)
59
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
60
+ raise ValueError("benchmark.repeats must be a positive integer")
61
+ return value
62
+
63
+
64
+ def build_plan(
65
+ config: VTuneConfig, run: Mapping[str, object], endpoint: str, artifacts: Path,
66
+ ) -> GuideLLMPlan:
67
+ name = run["name"]
68
+ request_format = run.get("request_format", "/v1/completions")
69
+ if not isinstance(request_format, str) or not request_format.strip():
70
+ raise ValueError("benchmark request_format must be a non-empty string")
71
+ directory = Path(artifacts) / str(name)
72
+ json_path = directory / "results.json"
73
+ argv = [
74
+ "guidellm", "run", "--backend",
75
+ _serialize({"kind": "openai_http", "target": endpoint,
76
+ "model": config.model.path, "request_format": request_format}),
77
+ "--profile", _serialize(_mapping(run.get("profile"), "profile")),
78
+ ]
79
+ for option, label in (("constraints", "constraint"), ("data", "data")):
80
+ values = run.get(option, [])
81
+ if not isinstance(values, list):
82
+ raise ValueError(f"benchmark run '{option}' must be a list")
83
+ for value in values:
84
+ if label == "constraint" and isinstance(value, dict) and value.get("kind") == "max_duration":
85
+ value = {**value, "seconds": normalize_durations(value.get("seconds"))}
86
+ argv.extend((f"--{label}", _serialize(_mapping(value, label))))
87
+ argv.extend(("--output", f"kind=json,path={json_path}",
88
+ "--disable-console-interactive"))
89
+ return GuideLLMPlan(str(name), tuple(argv), directory, json_path,
90
+ directory / "benchmark.log")
91
+
92
+
93
+ def parse_result(path: Path, run_name: str) -> BenchmarkResult:
94
+ source = Path(path)
95
+ try:
96
+ root = _mapping(json.loads(source.read_text(encoding="utf-8")), "result")
97
+ except (OSError, UnicodeError, json.JSONDecodeError) as error:
98
+ raise ValueError(f"Cannot read GuideLLM JSON result: {error}") from error
99
+ metadata = _mapping(root.get("metadata"), "metadata")
100
+ version = metadata.get("guidellm_version")
101
+ if not isinstance(version, str) or not version.strip():
102
+ raise ValueError("GuideLLM result is missing its version")
103
+ benchmarks = root.get("benchmarks")
104
+ if not isinstance(benchmarks, list) or not benchmarks:
105
+ raise ValueError("GuideLLM result must contain benchmarks")
106
+ workloads = tuple(
107
+ WorkloadResult(index, _mapping(item.get("config"), f"benchmark {index} config"),
108
+ _mapping(item.get("metrics"), f"benchmark {index} metrics"))
109
+ for index, value in enumerate(benchmarks)
110
+ for item in (_mapping(value, f"benchmark {index}"),)
111
+ )
112
+ return BenchmarkResult(run_name, "guidellm", version, workloads, source)
113
+
114
+
115
+ def _serialize(values: Mapping[str, object]) -> str:
116
+ kind = values.get("kind")
117
+ if not isinstance(kind, str) or not kind.strip():
118
+ raise ValueError("GuideLLM options require a non-empty 'kind'")
119
+ return ",".join(pair for key, value in values.items() for pair in _pairs(key, value))
120
+
121
+
122
+ def _pairs(path: str, value: object) -> tuple[str, ...]:
123
+ if isinstance(value, Mapping):
124
+ return tuple(pair for key, item in value.items()
125
+ for pair in _pairs(f"{path}.{key}", item))
126
+ if isinstance(value, list | tuple):
127
+ return tuple(pair for index, item in enumerate(value)
128
+ for pair in _pairs(f"{path}[{index}]", item))
129
+ if isinstance(value, bool):
130
+ rendered = str(value).lower()
131
+ if isinstance(value, str | int | float) and not isinstance(value, bool):
132
+ rendered = str(value)
133
+ elif value is None:
134
+ rendered = "null"
135
+ elif not isinstance(value, bool):
136
+ raise ValueError(f"Unsupported GuideLLM value at '{path}'")
137
+ if "," in rendered:
138
+ raise ValueError(f"GuideLLM string values cannot contain commas: '{path}'")
139
+ return (f"{path}={rendered}",)
140
+
141
+
142
+ def _mapping(value: object, label: str) -> Mapping[str, object]:
143
+ if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
144
+ raise ValueError(f"GuideLLM {label} must be an object")
145
+ return value
@@ -0,0 +1,67 @@
1
+ """Duration parsing and GuideLLM timeout estimation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ import re
7
+
8
+ _DURATION = re.compile(r"^(\d+(?:\.\d+)?)\s*([smh]?)$")
9
+ _UNITS = {"": 1, "s": 1, "m": 60, "h": 3600}
10
+
11
+
12
+ def parse_duration(value: object, label: str = "duration") -> float:
13
+ if isinstance(value, bool):
14
+ raise ValueError(f"'{label}' must be a positive duration")
15
+ if isinstance(value, int | float):
16
+ seconds = float(value)
17
+ elif isinstance(value, str) and (match := _DURATION.fullmatch(value.strip().lower())):
18
+ seconds = float(match.group(1)) * _UNITS[match.group(2)]
19
+ else:
20
+ raise ValueError(f"'{label}' must use seconds, '30s', '2m', or '1h'")
21
+ if seconds <= 0:
22
+ raise ValueError(f"'{label}' must be positive")
23
+ return seconds
24
+
25
+
26
+ def timeout_for_run(run: Mapping[str, object], configured: object = "auto") -> float:
27
+ """Return an explicit timeout or estimate duration plus a safety margin."""
28
+ if configured != "auto":
29
+ return parse_duration(configured, "timeouts.benchmark")
30
+ duration = _duration_constraint(run)
31
+ if duration is None:
32
+ return 180.0
33
+ expected = duration * _strategy_count(run.get("profile"))
34
+ return expected + max(30.0, expected * 0.25)
35
+
36
+
37
+ def normalize_durations(value: object) -> object:
38
+ """Convert duration strings recursively before passing values to GuideLLM."""
39
+ if isinstance(value, list):
40
+ return [normalize_durations(item) for item in value]
41
+ if isinstance(value, str):
42
+ return parse_duration(value)
43
+ return value
44
+
45
+
46
+ def _duration_constraint(run: Mapping[str, object]) -> float | None:
47
+ constraints = run.get("constraints", [])
48
+ if not isinstance(constraints, list):
49
+ return None
50
+ for constraint in constraints:
51
+ if isinstance(constraint, Mapping) and constraint.get("kind") == "max_duration":
52
+ seconds = constraint.get("seconds")
53
+ if isinstance(seconds, list):
54
+ return sum(parse_duration(item, "max_duration.seconds") for item in seconds)
55
+ return parse_duration(seconds, "max_duration.seconds")
56
+ return None
57
+
58
+
59
+ def _strategy_count(profile: object) -> int:
60
+ if not isinstance(profile, Mapping):
61
+ return 1
62
+ for key in ("streams", "rates"):
63
+ value = profile.get(key)
64
+ if isinstance(value, list):
65
+ return max(1, len(value))
66
+ size = profile.get("sweep_size")
67
+ return size if isinstance(size, int) and size > 0 else 1
@@ -0,0 +1,86 @@
1
+ """Command-line entry point for vTune."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import asyncio
7
+ from collections.abc import Sequence
8
+ from pathlib import Path
9
+ import sys
10
+
11
+ from vtune.config.errors import ConfigError
12
+ from vtune.config.loader import load_config
13
+ from vtune.cli_options import CLIUsageError, validate_cli_options
14
+ from vtune.lifecycle import load_retry_plan
15
+ from vtune.orchestrator import Orchestrator
16
+ from vtune.reproduction.display import reproduce_trial
17
+ from vtune.reproduction.export import export_vllm_command
18
+ from vtune.terminal import with_debug_logging
19
+
20
+
21
+ def build_parser() -> argparse.ArgumentParser:
22
+ parser = argparse.ArgumentParser(
23
+ prog="vtune", description="Experiment with vLLM serving configurations."
24
+ )
25
+ parser.add_argument(
26
+ "action", nargs="?", choices=("validate", "export", "reproduce", "retry"),
27
+ help="Post-run action; omit to start a new experiment.")
28
+ parser.add_argument("--config", "-c", metavar="YAML",
29
+ help="Experiment YAML for a new run or validation.")
30
+ parser.add_argument("--run", type=Path, metavar="DIRECTORY",
31
+ help="Existing immutable run directory.")
32
+ parser.add_argument("--trial", action="append", metavar="ID",
33
+ help="Trial ID; repeat the option when retrying several trials.")
34
+ parser.add_argument("--verbose", action="store_true",
35
+ help="Override logging.level with DEBUG and stream child logs.")
36
+ return parser
37
+
38
+
39
+ def main(argv: Sequence[str] | None = None) -> int:
40
+ args = build_parser().parse_args(argv)
41
+ try:
42
+ validate_cli_options(args)
43
+ if args.action == "export":
44
+ print(export_vllm_command(args.run, args.trial[0]))
45
+ return 0
46
+ if args.action == "reproduce":
47
+ print(reproduce_trial(args.run, args.trial[0]))
48
+ return 0
49
+ if args.action == "retry":
50
+ plan = load_retry_plan(args.run, args.trial)
51
+ for warning in plan.warnings:
52
+ print(f"Integrity warning: {warning}", file=sys.stderr)
53
+ retry_config = with_debug_logging(plan.config) if args.verbose else plan.config
54
+ outcome = asyncio.run(Orchestrator(
55
+ retry_config, plan.trials, plan.source_run_id, plan.sources,
56
+ ).run())
57
+ else:
58
+ config = load_config(args.config)
59
+ config = with_debug_logging(config) if args.verbose else config
60
+ if args.action == "validate":
61
+ Orchestrator(config).validate()
62
+ print(f"Configuration valid: {config.experiment.name}")
63
+ print(f"Model: {config.model.path}")
64
+ return 0
65
+ outcome = asyncio.run(Orchestrator(config).run())
66
+ except CLIUsageError as error:
67
+ print(f"Command error: {error}", file=sys.stderr)
68
+ return 2
69
+ except ConfigError as error:
70
+ print(f"Configuration error: {error}", file=sys.stderr)
71
+ return 2
72
+ except (OSError, TypeError, ValueError) as error:
73
+ print(f"Experiment error: {error}", file=sys.stderr)
74
+ return 1
75
+ except KeyboardInterrupt:
76
+ print("Experiment interrupted", file=sys.stderr)
77
+ return 130
78
+ print(outcome.summary)
79
+ if outcome.status == "interrupted":
80
+ return 130
81
+ completed = any(report.status.value == "completed" for report in outcome.trials)
82
+ return 0 if completed else 1
83
+
84
+
85
+ if __name__ == "__main__":
86
+ raise SystemExit(main())