tuning-client 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tuning_client-0.1.0/LICENSE +21 -0
- tuning_client-0.1.0/MANIFEST.in +10 -0
- tuning_client-0.1.0/PKG-INFO +199 -0
- tuning_client-0.1.0/README.md +176 -0
- tuning_client-0.1.0/pyproject.toml +30 -0
- tuning_client-0.1.0/setup.cfg +4 -0
- tuning_client-0.1.0/tuning_client/__init__.py +4 -0
- tuning_client-0.1.0/tuning_client/sdk.py +344 -0
- tuning_client-0.1.0/tuning_client.egg-info/SOURCES.txt +31 -0
- tuning_client-0.1.0/tuning_demo/__init__.py +1 -0
- tuning_client-0.1.0/tuning_demo/__main__.py +240 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/LICENSE-FLAML.txt +21 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/LICENSE-Ray.txt +177 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/README.md +135 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/THIRD_PARTY.md +49 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/__init__.py +32 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/asha.py +32 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/flaml.py +70 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/median.py +41 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/parallel.py +107 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/pb2.py +94 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/pb2_utils.py +139 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/pbt.py +60 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/population.py +33 -0
- tuning_client-0.1.0/tuning_demo/benchmarking/runner.py +167 -0
- tuning_client-0.1.0/tuning_demo/budgets.py +38 -0
- tuning_client-0.1.0/tuning_demo/datasets.py +98 -0
- tuning_client-0.1.0/tuning_demo/stats.py +319 -0
- tuning_client-0.1.0/tuning_demo/timing.py +80 -0
- tuning_client-0.1.0/tuning_demo/training.py +239 -0
- tuning_client-0.1.0/tuning_examples/__init__.py +1 -0
- tuning_client-0.1.0/tuning_examples/custom.py +78 -0
- tuning_client-0.1.0/tuning_examples/telemetry.py +26 -0
- tuning_client-0.1.0/tuning_examples/toys.py +123 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tuning Client contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Begin from an empty source manifest, then allow only the public distribution.
|
|
2
|
+
global-exclude *
|
|
3
|
+
include pyproject.toml MANIFEST.in README.md LICENSE*
|
|
4
|
+
recursive-include tuning_client *.py
|
|
5
|
+
recursive-include tuning_demo *.py
|
|
6
|
+
recursive-include tuning_examples *.py
|
|
7
|
+
include tuning_demo/benchmarking/README.md
|
|
8
|
+
include tuning_demo/benchmarking/THIRD_PARTY.md
|
|
9
|
+
include tuning_demo/benchmarking/LICENSE-Ray.txt
|
|
10
|
+
include tuning_demo/benchmarking/LICENSE-FLAML.txt
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tuning-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service
|
|
5
|
+
License-Expression: MIT AND Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://hyperopt.dev
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: tuning_demo/benchmarking/LICENSE-FLAML.txt
|
|
11
|
+
License-File: tuning_demo/benchmarking/LICENSE-Ray.txt
|
|
12
|
+
License-File: tuning_demo/benchmarking/THIRD_PARTY.md
|
|
13
|
+
Requires-Dist: httpx<1,>=0.27
|
|
14
|
+
Requires-Dist: websockets<18,>=15
|
|
15
|
+
Provides-Extra: demo
|
|
16
|
+
Requires-Dist: numpy<3,>=1.26; extra == "demo"
|
|
17
|
+
Requires-Dist: scikit-learn<2,>=1.5; extra == "demo"
|
|
18
|
+
Requires-Dist: torch<3,>=2.5; extra == "demo"
|
|
19
|
+
Provides-Extra: custom
|
|
20
|
+
Requires-Dist: pandas<3,>=2.2; extra == "custom"
|
|
21
|
+
Requires-Dist: pyarrow<24,>=18; extra == "custom"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# Tuning Client Python SDK and demo
|
|
25
|
+
|
|
26
|
+
Tuning Client is the Python SDK and demo for the
|
|
27
|
+
[hyperopt.dev AutoML service](https://hyperopt.dev).
|
|
28
|
+
|
|
29
|
+
Use the Python SDK to request tuning suggestions while keeping model training,
|
|
30
|
+
datasets, and model weights on your computer. The optional demo trains small
|
|
31
|
+
PyTorch models and compares tuning strategies on classification, regression,
|
|
32
|
+
and teaching datasets.
|
|
33
|
+
|
|
34
|
+
**Code provenance:** The SDK and demo code were generated with ChatGPT and
|
|
35
|
+
reviewed. Review does not guarantee correctness; evaluate the code and benchmark
|
|
36
|
+
results for your use case. The demos are examples, not production ML benchmarks.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
Requires Python 3.11 or newer. Install the SDK alone or include the demo:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
python -m pip install tuning-client
|
|
44
|
+
python -m pip install 'tuning-client[demo]'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Run the demo
|
|
48
|
+
|
|
49
|
+
Create an account using the service's signup page and verify your email. Replace
|
|
50
|
+
`https://YOUR_API` below with the service URL provided to you:
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
python -m tuning_demo --api-url https://YOUR_API --email you@example.com --dataset digits
|
|
54
|
+
# The installed command is equivalent:
|
|
55
|
+
tuning-client-demo --api-url https://YOUR_API --email you@example.com --dataset iris
|
|
56
|
+
python -m tuning_demo --list-datasets
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
|
|
60
|
+
or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
|
|
61
|
+
`--api-url` each time. Connections use HTTPS and verified TLS.
|
|
62
|
+
|
|
63
|
+
Service effort defaults to **medium**. Use `--effort easy` for lower effort;
|
|
64
|
+
the observation budget stays the same. `--interactive` offers another dataset
|
|
65
|
+
after each run. Sessions created by the demo are released automatically on
|
|
66
|
+
completion or failure. `--session-id ID` uses an existing unused session that
|
|
67
|
+
you remain responsible for releasing; `--keep-session` retains a new session.
|
|
68
|
+
|
|
69
|
+
Results are saved in `./results/`. Downloaded datasets are cached in
|
|
70
|
+
`~/.cache/tuning-client/`. Override these with `--output-dir` and `--data-dir`.
|
|
71
|
+
Optional usage events include a run ID, session ID, workload category, and
|
|
72
|
+
progress event; use `--no-telemetry` to disable them. Training data, file paths,
|
|
73
|
+
and model weights are not included in those events.
|
|
74
|
+
|
|
75
|
+
## SDK usage
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import os
|
|
79
|
+
from tuning_client import TuningClient
|
|
80
|
+
|
|
81
|
+
with TuningClient(os.environ['COMPUTE_API_URL'],
|
|
82
|
+
access_token=os.environ['COMPUTE_ACCESS_TOKEN']) as api:
|
|
83
|
+
session_id = api.create_session()
|
|
84
|
+
try:
|
|
85
|
+
stream = api.tuning_session(session_id)
|
|
86
|
+
try:
|
|
87
|
+
configurations = [{'learning_rate': rate} for rate in
|
|
88
|
+
[0.001, 0.002, 0.003, 0.004, 0.005,
|
|
89
|
+
0.006, 0.007, 0.008, 0.009, 0.010]]
|
|
90
|
+
stream.initialize(configurations, max_steps=10, seed=42)
|
|
91
|
+
suggestion = stream.suggest()
|
|
92
|
+
# Train the suggested configuration locally, then report validation loss:
|
|
93
|
+
# stream.report(suggestion['idx'], validation_loss)
|
|
94
|
+
# Continue with suggest/report or report_and_suggest as needed.
|
|
95
|
+
finally:
|
|
96
|
+
stream.close()
|
|
97
|
+
finally:
|
|
98
|
+
api.terminate_session(session_id)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`initialize` and `reset` default to `effort="medium"`. A report records one
|
|
102
|
+
observation; `max_steps` limits observations per model, not optimizer updates.
|
|
103
|
+
Use `status()` to inspect progress and rankings. `reset(...)` starts a fresh
|
|
104
|
+
run after the previous run completes. `api.list_sessions()` lists your sessions;
|
|
105
|
+
`api.wait_for_termination(session_id)` waits for a release to complete.
|
|
106
|
+
|
|
107
|
+
## Datasets and training budgets
|
|
108
|
+
|
|
109
|
+
Each demo starts with ten sampled model configurations and allows up to 50
|
|
110
|
+
validation observations across them, with at most ten observations per model.
|
|
111
|
+
After tuning, all models finish to the same update cap for comparison.
|
|
112
|
+
Classification ranks use validation cross-entropy; regression ranks use MSE in
|
|
113
|
+
original target units. Lower is better. Test data is used only for final reporting.
|
|
114
|
+
|
|
115
|
+
| Dataset | Updates per observation | Maximum updates per model | Source |
|
|
116
|
+
| --- | ---: | ---: | --- |
|
|
117
|
+
| `iris` | 10 | 100 | scikit-learn |
|
|
118
|
+
| `wine` | 200 | 2,000 | scikit-learn |
|
|
119
|
+
| `breast-cancer` | 30 | 300 | scikit-learn |
|
|
120
|
+
| `digits` | 60 | 600 | scikit-learn |
|
|
121
|
+
| `glass` | 100 | 1,000 | OpenML dataset 41 |
|
|
122
|
+
| `vehicle` | 60 | 600 | OpenML dataset 54 |
|
|
123
|
+
| `diabetes` | 10 | 100 | scikit-learn |
|
|
124
|
+
| `friedman` | 200 | 2,000 | Generated regression data |
|
|
125
|
+
| `linear-regression` | 200 | 2,000 | Generated regression data |
|
|
126
|
+
| `toy-lm` | 30 | 300 | Generated language teaching task |
|
|
127
|
+
| `toy-vla` | 200 | 2,000 | Generated vision/language teaching task |
|
|
128
|
+
|
|
129
|
+
Glass and vehicle download on first use. Ordinary datasets use a 70/15/15
|
|
130
|
+
train/validation/test split; wine uses 55/30/15. Identical language contexts are
|
|
131
|
+
grouped into one split. Preprocessing is fitted on training data only.
|
|
132
|
+
|
|
133
|
+
`--block-steps N` sets updates per observation and a maximum of `10*N` updates.
|
|
134
|
+
The defaults are bounded workload profiles, not guarantees of convergence.
|
|
135
|
+
Version 2 uses loss-based classification ranks; these are not comparable with
|
|
136
|
+
version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
|
|
137
|
+
`--split-seed`, and `--training-seed` allow independent overrides.
|
|
138
|
+
|
|
139
|
+
## Repeated benchmarks
|
|
140
|
+
|
|
141
|
+
```sh
|
|
142
|
+
python -m tuning_demo.stats --api-url https://YOUR_API --email you@example.com \
|
|
143
|
+
--datasets iris wine --runs 2
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Without dataset/run options, this runs all eleven datasets ten times, starting
|
|
147
|
+
at seed 42. Each trial starts with fresh models and tuning state. Sessions are
|
|
148
|
+
reused per dataset; `--session-scope run` reuses one session across datasets.
|
|
149
|
+
Service benchmarks run sequentially.
|
|
150
|
+
|
|
151
|
+
Local reference schedulers need no service account:
|
|
152
|
+
|
|
153
|
+
```sh
|
|
154
|
+
python -m tuning_demo --scheduler asha --dataset digits
|
|
155
|
+
python -m tuning_demo.stats --scheduler flaml --datasets iris wine --runs 2 --workers 2
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Available schedulers are `service` (default), `asha`, `median`, `pbt`, `pb2`,
|
|
159
|
+
and `flaml`. `--workers` applies only to local reference schedulers; it cannot
|
|
160
|
+
parallelize service benchmarks. Reference statistics default to available CPU
|
|
161
|
+
cores, capped by trial count; use `--workers 1` to limit resource use.
|
|
162
|
+
See [benchmark details](tuning_demo/benchmarking/README.md) for algorithm settings.
|
|
163
|
+
|
|
164
|
+
Results include configurations, seeds, observation histories, timings, losses,
|
|
165
|
+
and rankings at the tuning cutoff and after final training. Statistics report
|
|
166
|
+
the mean final rank of the model selected at the cutoff and the best final rank
|
|
167
|
+
among models already capped at that cutoff. Ties receive ranks such as 1, 1, 3;
|
|
168
|
+
unavailable measurements and failed trials do not count as zero. A failed service
|
|
169
|
+
trial skips the remaining trials for that dataset. The command exits nonzero
|
|
170
|
+
on trial or cleanup failure and preserves partial results.
|
|
171
|
+
|
|
172
|
+
Timing includes client training and service round trips; it is not a pure
|
|
173
|
+
measurement of service compute. The service ranking uses latest reported losses,
|
|
174
|
+
not predicted fully trained losses. PBT/PB2 compare evolved checkpoint lineages,
|
|
175
|
+
not the original fixed configurations.
|
|
176
|
+
|
|
177
|
+
## Custom datasets
|
|
178
|
+
|
|
179
|
+
```sh
|
|
180
|
+
python -m pip install 'tuning-client[demo,custom]'
|
|
181
|
+
python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.csv \
|
|
182
|
+
--target label --task classification --drop-columns id
|
|
183
|
+
python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.parquet \
|
|
184
|
+
--target price --task regression
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Use one row per independent example, unique column names, a non-missing scalar
|
|
188
|
+
target, and numeric or categorical input columns. Drop identifiers and columns
|
|
189
|
+
that leak the target. Custom datasets default to 100 updates per observation.
|
|
190
|
+
Files stay local. Use an appropriate split outside this demo for time series,
|
|
191
|
+
grouped observations, or other data that requires special leakage controls.
|
|
192
|
+
|
|
193
|
+
## License and third-party notices
|
|
194
|
+
|
|
195
|
+
Original SDK and demo code is licensed under the [MIT License](LICENSE).
|
|
196
|
+
|
|
197
|
+
The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
|
|
198
|
+
and FLAML (MIT). Their license texts and attribution are included in the package;
|
|
199
|
+
see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# Tuning Client Python SDK and demo
|
|
2
|
+
|
|
3
|
+
Tuning Client is the Python SDK and demo for the
|
|
4
|
+
[hyperopt.dev AutoML service](https://hyperopt.dev).
|
|
5
|
+
|
|
6
|
+
Use the Python SDK to request tuning suggestions while keeping model training,
|
|
7
|
+
datasets, and model weights on your computer. The optional demo trains small
|
|
8
|
+
PyTorch models and compares tuning strategies on classification, regression,
|
|
9
|
+
and teaching datasets.
|
|
10
|
+
|
|
11
|
+
**Code provenance:** The SDK and demo code were generated with ChatGPT and
|
|
12
|
+
reviewed. Review does not guarantee correctness; evaluate the code and benchmark
|
|
13
|
+
results for your use case. The demos are examples, not production ML benchmarks.
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
Requires Python 3.11 or newer. Install the SDK alone or include the demo:
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
python -m pip install tuning-client
|
|
21
|
+
python -m pip install 'tuning-client[demo]'
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Run the demo
|
|
25
|
+
|
|
26
|
+
Create an account using the service's signup page and verify your email. Replace
|
|
27
|
+
`https://YOUR_API` below with the service URL provided to you:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
python -m tuning_demo --api-url https://YOUR_API --email you@example.com --dataset digits
|
|
31
|
+
# The installed command is equivalent:
|
|
32
|
+
tuning-client-demo --api-url https://YOUR_API --email you@example.com --dataset iris
|
|
33
|
+
python -m tuning_demo --list-datasets
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
|
|
37
|
+
or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
|
|
38
|
+
`--api-url` each time. Connections use HTTPS and verified TLS.
|
|
39
|
+
|
|
40
|
+
Service effort defaults to **medium**. Use `--effort easy` for lower effort;
|
|
41
|
+
the observation budget stays the same. `--interactive` offers another dataset
|
|
42
|
+
after each run. Sessions created by the demo are released automatically on
|
|
43
|
+
completion or failure. `--session-id ID` uses an existing unused session that
|
|
44
|
+
you remain responsible for releasing; `--keep-session` retains a new session.
|
|
45
|
+
|
|
46
|
+
Results are saved in `./results/`. Downloaded datasets are cached in
|
|
47
|
+
`~/.cache/tuning-client/`. Override these with `--output-dir` and `--data-dir`.
|
|
48
|
+
Optional usage events include a run ID, session ID, workload category, and
|
|
49
|
+
progress event; use `--no-telemetry` to disable them. Training data, file paths,
|
|
50
|
+
and model weights are not included in those events.
|
|
51
|
+
|
|
52
|
+
## SDK usage
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import os
|
|
56
|
+
from tuning_client import TuningClient
|
|
57
|
+
|
|
58
|
+
with TuningClient(os.environ['COMPUTE_API_URL'],
|
|
59
|
+
access_token=os.environ['COMPUTE_ACCESS_TOKEN']) as api:
|
|
60
|
+
session_id = api.create_session()
|
|
61
|
+
try:
|
|
62
|
+
stream = api.tuning_session(session_id)
|
|
63
|
+
try:
|
|
64
|
+
configurations = [{'learning_rate': rate} for rate in
|
|
65
|
+
[0.001, 0.002, 0.003, 0.004, 0.005,
|
|
66
|
+
0.006, 0.007, 0.008, 0.009, 0.010]]
|
|
67
|
+
stream.initialize(configurations, max_steps=10, seed=42)
|
|
68
|
+
suggestion = stream.suggest()
|
|
69
|
+
# Train the suggested configuration locally, then report validation loss:
|
|
70
|
+
# stream.report(suggestion['idx'], validation_loss)
|
|
71
|
+
# Continue with suggest/report or report_and_suggest as needed.
|
|
72
|
+
finally:
|
|
73
|
+
stream.close()
|
|
74
|
+
finally:
|
|
75
|
+
api.terminate_session(session_id)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`initialize` and `reset` default to `effort="medium"`. A report records one
|
|
79
|
+
observation; `max_steps` limits observations per model, not optimizer updates.
|
|
80
|
+
Use `status()` to inspect progress and rankings. `reset(...)` starts a fresh
|
|
81
|
+
run after the previous run completes. `api.list_sessions()` lists your sessions;
|
|
82
|
+
`api.wait_for_termination(session_id)` waits for a release to complete.
|
|
83
|
+
|
|
84
|
+
## Datasets and training budgets
|
|
85
|
+
|
|
86
|
+
Each demo starts with ten sampled model configurations and allows up to 50
|
|
87
|
+
validation observations across them, with at most ten observations per model.
|
|
88
|
+
After tuning, all models finish to the same update cap for comparison.
|
|
89
|
+
Classification ranks use validation cross-entropy; regression ranks use MSE in
|
|
90
|
+
original target units. Lower is better. Test data is used only for final reporting.
|
|
91
|
+
|
|
92
|
+
| Dataset | Updates per observation | Maximum updates per model | Source |
|
|
93
|
+
| --- | ---: | ---: | --- |
|
|
94
|
+
| `iris` | 10 | 100 | scikit-learn |
|
|
95
|
+
| `wine` | 200 | 2,000 | scikit-learn |
|
|
96
|
+
| `breast-cancer` | 30 | 300 | scikit-learn |
|
|
97
|
+
| `digits` | 60 | 600 | scikit-learn |
|
|
98
|
+
| `glass` | 100 | 1,000 | OpenML dataset 41 |
|
|
99
|
+
| `vehicle` | 60 | 600 | OpenML dataset 54 |
|
|
100
|
+
| `diabetes` | 10 | 100 | scikit-learn |
|
|
101
|
+
| `friedman` | 200 | 2,000 | Generated regression data |
|
|
102
|
+
| `linear-regression` | 200 | 2,000 | Generated regression data |
|
|
103
|
+
| `toy-lm` | 30 | 300 | Generated language teaching task |
|
|
104
|
+
| `toy-vla` | 200 | 2,000 | Generated vision/language teaching task |
|
|
105
|
+
|
|
106
|
+
Glass and vehicle download on first use. Ordinary datasets use a 70/15/15
|
|
107
|
+
train/validation/test split; wine uses 55/30/15. Identical language contexts are
|
|
108
|
+
grouped into one split. Preprocessing is fitted on training data only.
|
|
109
|
+
|
|
110
|
+
`--block-steps N` sets updates per observation and a maximum of `10*N` updates.
|
|
111
|
+
The defaults are bounded workload profiles, not guarantees of convergence.
|
|
112
|
+
Version 2 uses loss-based classification ranks; these are not comparable with
|
|
113
|
+
version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
|
|
114
|
+
`--split-seed`, and `--training-seed` allow independent overrides.
|
|
115
|
+
|
|
116
|
+
## Repeated benchmarks
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
python -m tuning_demo.stats --api-url https://YOUR_API --email you@example.com \
|
|
120
|
+
--datasets iris wine --runs 2
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Without dataset/run options, this runs all eleven datasets ten times, starting
|
|
124
|
+
at seed 42. Each trial starts with fresh models and tuning state. Sessions are
|
|
125
|
+
reused per dataset; `--session-scope run` reuses one session across datasets.
|
|
126
|
+
Service benchmarks run sequentially.
|
|
127
|
+
|
|
128
|
+
Local reference schedulers need no service account:
|
|
129
|
+
|
|
130
|
+
```sh
|
|
131
|
+
python -m tuning_demo --scheduler asha --dataset digits
|
|
132
|
+
python -m tuning_demo.stats --scheduler flaml --datasets iris wine --runs 2 --workers 2
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Available schedulers are `service` (default), `asha`, `median`, `pbt`, `pb2`,
|
|
136
|
+
and `flaml`. `--workers` applies only to local reference schedulers; it cannot
|
|
137
|
+
parallelize service benchmarks. Reference statistics default to available CPU
|
|
138
|
+
cores, capped by trial count; use `--workers 1` to limit resource use.
|
|
139
|
+
See [benchmark details](tuning_demo/benchmarking/README.md) for algorithm settings.
|
|
140
|
+
|
|
141
|
+
Results include configurations, seeds, observation histories, timings, losses,
|
|
142
|
+
and rankings at the tuning cutoff and after final training. Statistics report
|
|
143
|
+
the mean final rank of the model selected at the cutoff and the best final rank
|
|
144
|
+
among models already capped at that cutoff. Ties receive ranks such as 1, 1, 3;
|
|
145
|
+
unavailable measurements and failed trials do not count as zero. A failed service
|
|
146
|
+
trial skips the remaining trials for that dataset. The command exits nonzero
|
|
147
|
+
on trial or cleanup failure and preserves partial results.
|
|
148
|
+
|
|
149
|
+
Timing includes client training and service round trips; it is not a pure
|
|
150
|
+
measurement of service compute. The service ranking uses latest reported losses,
|
|
151
|
+
not predicted fully trained losses. PBT/PB2 compare evolved checkpoint lineages,
|
|
152
|
+
not the original fixed configurations.
|
|
153
|
+
|
|
154
|
+
## Custom datasets
|
|
155
|
+
|
|
156
|
+
```sh
|
|
157
|
+
python -m pip install 'tuning-client[demo,custom]'
|
|
158
|
+
python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.csv \
|
|
159
|
+
--target label --task classification --drop-columns id
|
|
160
|
+
python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.parquet \
|
|
161
|
+
--target price --task regression
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Use one row per independent example, unique column names, a non-missing scalar
|
|
165
|
+
target, and numeric or categorical input columns. Drop identifiers and columns
|
|
166
|
+
that leak the target. Custom datasets default to 100 updates per observation.
|
|
167
|
+
Files stay local. Use an appropriate split outside this demo for time series,
|
|
168
|
+
grouped observations, or other data that requires special leakage controls.
|
|
169
|
+
|
|
170
|
+
## License and third-party notices
|
|
171
|
+
|
|
172
|
+
Original SDK and demo code is licensed under the [MIT License](LICENSE).
|
|
173
|
+
|
|
174
|
+
The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
|
|
175
|
+
and FLAML (MIT). Their license texts and attribution are included in the package;
|
|
176
|
+
see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0.3"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tuning-client"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT AND Apache-2.0"
|
|
11
|
+
license-files = ["LICENSE*", "tuning_demo/benchmarking/LICENSE-*.txt", "tuning_demo/benchmarking/THIRD_PARTY.md"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
dependencies = ["httpx>=0.27,<1", "websockets>=15,<18"]
|
|
14
|
+
|
|
15
|
+
[project.urls]
|
|
16
|
+
Homepage = "https://hyperopt.dev"
|
|
17
|
+
|
|
18
|
+
[project.optional-dependencies]
|
|
19
|
+
demo = ["numpy>=1.26,<3", "scikit-learn>=1.5,<2", "torch>=2.5,<3"]
|
|
20
|
+
custom = ["pandas>=2.2,<3", "pyarrow>=18,<24"]
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
tuning-client-demo = "tuning_demo.__main__:main"
|
|
24
|
+
|
|
25
|
+
[tool.setuptools]
|
|
26
|
+
packages = ["tuning_client", "tuning_demo", "tuning_demo.benchmarking", "tuning_examples"]
|
|
27
|
+
include-package-data = false
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.package-data]
|
|
30
|
+
"tuning_demo.benchmarking" = ["README.md", "THIRD_PARTY.md", "LICENSE-Ray.txt", "LICENSE-FLAML.txt"]
|