tuning-client 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tuning_client-0.2.0 → tuning_client-0.2.2}/PKG-INFO +83 -9
- {tuning_client-0.2.0 → tuning_client-0.2.2}/README.md +79 -8
- {tuning_client-0.2.0 → tuning_client-0.2.2}/pyproject.toml +2 -1
- tuning_client-0.2.2/tuning_client/__init__.py +6 -0
- tuning_client-0.2.2/tuning_client/_local_mlflow.py +187 -0
- tuning_client-0.2.2/tuning_client/mlflow.py +103 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_client.egg-info/SOURCES.txt +2 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/__main__.py +66 -1
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/THIRD_PARTY.md +2 -2
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/budgets.py +4 -2
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/stats.py +3 -1
- tuning_client-0.2.0/tuning_client/__init__.py +0 -4
- {tuning_client-0.2.0 → tuning_client-0.2.2}/LICENSE +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/MANIFEST.in +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/setup.cfg +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_client/sdk.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/__init__.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/LICENSE-FLAML.txt +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/LICENSE-Ray.txt +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/README.md +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/__init__.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/asha.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/flaml.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/median.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/parallel.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/pb2.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/pb2_utils.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/pbt.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/population.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/runner.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/datasets.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/timing.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/training.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/__init__.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/custom.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/telemetry.py +0 -0
- {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/toys.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tuning-client
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service
|
|
5
5
|
License-Expression: MIT AND Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://hyperopt.dev
|
|
@@ -12,6 +12,9 @@ License-File: tuning_demo/benchmarking/LICENSE-Ray.txt
|
|
|
12
12
|
License-File: tuning_demo/benchmarking/THIRD_PARTY.md
|
|
13
13
|
Requires-Dist: httpx<1,>=0.27
|
|
14
14
|
Requires-Dist: websockets<18,>=15
|
|
15
|
+
Provides-Extra: mlflow
|
|
16
|
+
Requires-Dist: tuning-mlflow==3.18.0.post1; extra == "mlflow"
|
|
17
|
+
Requires-Dist: filelock<5,>=3.12; extra == "mlflow"
|
|
15
18
|
Provides-Extra: demo
|
|
16
19
|
Requires-Dist: numpy<3,>=1.26; extra == "demo"
|
|
17
20
|
Requires-Dist: scikit-learn<2,>=1.5; extra == "demo"
|
|
@@ -49,11 +52,11 @@ selected by the default PyPI distribution:
|
|
|
49
52
|
|
|
50
53
|
```sh
|
|
51
54
|
python -m pip install 'torch>=2.5,<3' --index-url https://download.pytorch.org/whl/cpu
|
|
52
|
-
python -m pip install 'tuning-client[demo]'
|
|
55
|
+
python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
|
|
53
56
|
```
|
|
54
57
|
|
|
55
58
|
The installed CPU build satisfies the demo's PyTorch requirement. On macOS,
|
|
56
|
-
install `tuning-client[demo]` directly. See the
|
|
59
|
+
install `tuning-client[demo,mlflow]>=0.2.2` directly. See the
|
|
57
60
|
[PyTorch installation guide](https://pytorch.org/get-started/locally/) for other
|
|
58
61
|
platforms. If a failed download filled the disk, `python -m pip cache purge`
|
|
59
62
|
removes pip's cached downloads; `--no-cache-dir` avoids caching the next install.
|
|
@@ -71,6 +74,11 @@ tuning-client-demo --api-url https://hyperopt.dev --email you@example.com --data
|
|
|
71
74
|
python -m tuning_demo --list-datasets
|
|
72
75
|
```
|
|
73
76
|
|
|
77
|
+
The service demo first offers to start a local MLflow dashboard. Answer y to enable
|
|
78
|
+
tracking, or press Enter to continue. Use `--mlflow-local` to enable it explicitly
|
|
79
|
+
or `--no-mlflow` to skip the prompt. Noninteractive commands require
|
|
80
|
+
`--mlflow-local` or `--mlflow-tracking-uri` to enable tracking.
|
|
81
|
+
|
|
74
82
|
The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
|
|
75
83
|
or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
|
|
76
84
|
`--api-url` each time. Connections use HTTPS and verified TLS.
|
|
@@ -95,9 +103,7 @@ with TuningClient(os.environ['COMPUTE_API_URL'],
|
|
|
95
103
|
try:
|
|
96
104
|
stream = api.tuning_session(session_id)
|
|
97
105
|
try:
|
|
98
|
-
configurations = [{'learning_rate':
|
|
99
|
-
[0.001, 0.002, 0.003, 0.004, 0.005,
|
|
100
|
-
0.006, 0.007, 0.008, 0.009, 0.010]]
|
|
106
|
+
configurations = [{'learning_rate': 0.001 * index} for index in range(1, 21)]
|
|
101
107
|
stream.initialize(configurations, max_steps=10, seed=42)
|
|
102
108
|
suggestion = stream.suggest()
|
|
103
109
|
# Train the suggested configuration locally, then report validation loss:
|
|
@@ -127,7 +133,7 @@ original target units. Lower is better. Test data is used only for final reporti
|
|
|
127
133
|
| --- | ---: | ---: | --- |
|
|
128
134
|
| `iris` | 10 | 100 | scikit-learn |
|
|
129
135
|
| `wine` | 200 | 2,000 | scikit-learn |
|
|
130
|
-
| `breast-cancer` |
|
|
136
|
+
| `breast-cancer` | 60 | 600 | scikit-learn |
|
|
131
137
|
| `digits` | 60 | 600 | scikit-learn |
|
|
132
138
|
| `glass` | 100 | 1,000 | OpenML dataset 41 |
|
|
133
139
|
| `vehicle` | 60 | 600 | OpenML dataset 54 |
|
|
@@ -143,8 +149,10 @@ grouped into one split. Preprocessing is fitted on training data only.
|
|
|
143
149
|
|
|
144
150
|
`--block-steps N` sets updates per observation and a maximum of `10*N` updates.
|
|
145
151
|
The defaults are bounded workload profiles, not guarantees of convergence.
|
|
146
|
-
Version
|
|
147
|
-
|
|
152
|
+
Version 3 raises the breast-cancer cap from 300 to 600 updates (60 per
|
|
153
|
+
observation); compare new breast-cancer results only with runs using that cap.
|
|
154
|
+
Version 2 introduced loss-based classification ranks, which are not comparable
|
|
155
|
+
with version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
|
|
148
156
|
`--split-seed`, and `--training-seed` allow independent overrides.
|
|
149
157
|
|
|
150
158
|
## Repeated benchmarks
|
|
@@ -208,3 +216,69 @@ Original SDK and demo code is licensed under the [MIT License](LICENSE).
|
|
|
208
216
|
The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
|
|
209
217
|
and FLAML (MIT). Their license texts and attribution are included in the package;
|
|
210
218
|
see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
|
|
219
|
+
|
|
220
|
+
## Optional MLflow tracking
|
|
221
|
+
|
|
222
|
+
Install the dashboard dependencies with the demo:
|
|
223
|
+
|
|
224
|
+
```sh
|
|
225
|
+
python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
|
|
226
|
+
python -m tuning_demo --api-url https://hyperopt.dev --dataset breast-cancer
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
In an interactive terminal, the service demo asks permission to start a local
|
|
230
|
+
dashboard. Answering y starts it on 127.0.0.1 and enables tracking. Pressing
|
|
231
|
+
Enter declines. For scripts, use `--mlflow-local` to enable local startup or
|
|
232
|
+
`--no-mlflow` to disable tracking and the prompt. An existing tracking server
|
|
233
|
+
can be selected explicitly with `--mlflow-tracking-uri URL`. These options
|
|
234
|
+
are mutually exclusive. `--mlflow-experiment` names the experiment.
|
|
235
|
+
|
|
236
|
+
The SDK starts one background process, waits for readiness, and reuses that
|
|
237
|
+
instance on later invocations. It prefers local port 5001 and selects another
|
|
238
|
+
available port if needed. The server stays running after training so the printed
|
|
239
|
+
dashboard link continues working. Its SQLite database, artifacts, state and server log persist
|
|
240
|
+
under `~/.local/share/tuning-client/mlflow` on macOS/Linux, respecting
|
|
241
|
+
`XDG_DATA_HOME`. Windows uses `LOCALAPPDATA/tuning-client/mlflow`.
|
|
242
|
+
The local instance uses the same Python environment as the demo.
|
|
243
|
+
Install the extra before opting in. Python 3.11+ supports both packages.
|
|
244
|
+
|
|
245
|
+
Tracking logs the completed service result and prints one link to that tuning
|
|
246
|
+
job. The dashboard highlights the selected model, plots candidate validation
|
|
247
|
+
losses against training steps, and shows each candidate's training allocation.
|
|
248
|
+
Hyperparameters and other candidates expand on demand. Tracking stores candidate
|
|
249
|
+
configurations and scalar metrics in the selected tracking server.
|
|
250
|
+
|
|
251
|
+
The fork supplies the `mlflow` Python namespace. Install it in an environment
|
|
252
|
+
without upstream `mlflow`, `mlflow-skinny` or `mlflow-tracing`, which own
|
|
253
|
+
overlapping files. The SDK can also log to an explicitly configured upstream
|
|
254
|
+
tracking server. Tracking errors propagate, and the normal JSON result is
|
|
255
|
+
saved before result logging begins.
|
|
256
|
+
|
|
257
|
+
Each completed experiment gets a parent run and one child run per candidate.
|
|
258
|
+
The child tagged `tuning.best_run=true` is the binary's selection from
|
|
259
|
+
`service_at_budget.ranking`, without reranking after local completion.
|
|
260
|
+
`service_validation_error` records that frozen selection metric; lower is
|
|
261
|
+
better. `validation_error` records the original observed curve at optimizer
|
|
262
|
+
steps. `final_validation_error`, `train_error`, and `test_error` record the
|
|
263
|
+
separate fully trained evaluation once. The parent tag `tuning.final_best_idx`
|
|
264
|
+
preserves the fully trained validation winner, which can differ from the
|
|
265
|
+
binary's winner. All candidate runs have `tuning.run=true`.
|
|
266
|
+
|
|
267
|
+
For a custom SDK workflow, build the same result shape as the demo and log it:
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
from tuning_client import MLflowTracker, start_local_mlflow
|
|
271
|
+
|
|
272
|
+
# Invoke after your application has obtained the user's consent.
|
|
273
|
+
tracking_uri = start_local_mlflow()
|
|
274
|
+
tracker = MLflowTracker(tracking_uri, experiment_name="Tuning")
|
|
275
|
+
run_id = tracker.log_result(result)
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
`result` contains `models` (each with `idx`, `configuration`, `steps`, `error`,
|
|
279
|
+
and optional `train_error`/`test_error`), `history` (`idx`, `steps`, `error`),
|
|
280
|
+
`best_validation_idx`, and optionally `service_at_budget` with `ranking` entries
|
|
281
|
+
(`idx`, `error`) and `rounds`. Without a service ranking, the supplied
|
|
282
|
+
`best_validation_idx` is used. Reusing the same result object on the same
|
|
283
|
+
tracker does not log it twice; cross-process retry deduplication is not provided.
|
|
284
|
+
The tracker leaves fluent MLflow active runs untouched.
|
|
@@ -26,11 +26,11 @@ selected by the default PyPI distribution:
|
|
|
26
26
|
|
|
27
27
|
```sh
|
|
28
28
|
python -m pip install 'torch>=2.5,<3' --index-url https://download.pytorch.org/whl/cpu
|
|
29
|
-
python -m pip install 'tuning-client[demo]'
|
|
29
|
+
python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
The installed CPU build satisfies the demo's PyTorch requirement. On macOS,
|
|
33
|
-
install `tuning-client[demo]` directly. See the
|
|
33
|
+
install `tuning-client[demo,mlflow]>=0.2.2` directly. See the
|
|
34
34
|
[PyTorch installation guide](https://pytorch.org/get-started/locally/) for other
|
|
35
35
|
platforms. If a failed download filled the disk, `python -m pip cache purge`
|
|
36
36
|
removes pip's cached downloads; `--no-cache-dir` avoids caching the next install.
|
|
@@ -48,6 +48,11 @@ tuning-client-demo --api-url https://hyperopt.dev --email you@example.com --data
|
|
|
48
48
|
python -m tuning_demo --list-datasets
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
+
The service demo first offers to start a local MLflow dashboard. Answer y to enable
|
|
52
|
+
tracking, or press Enter to continue. Use `--mlflow-local` to enable it explicitly
|
|
53
|
+
or `--no-mlflow` to skip the prompt. Noninteractive commands require
|
|
54
|
+
`--mlflow-local` or `--mlflow-tracking-uri` to enable tracking.
|
|
55
|
+
|
|
51
56
|
The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
|
|
52
57
|
or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
|
|
53
58
|
`--api-url` each time. Connections use HTTPS and verified TLS.
|
|
@@ -72,9 +77,7 @@ with TuningClient(os.environ['COMPUTE_API_URL'],
|
|
|
72
77
|
try:
|
|
73
78
|
stream = api.tuning_session(session_id)
|
|
74
79
|
try:
|
|
75
|
-
configurations = [{'learning_rate':
|
|
76
|
-
[0.001, 0.002, 0.003, 0.004, 0.005,
|
|
77
|
-
0.006, 0.007, 0.008, 0.009, 0.010]]
|
|
80
|
+
configurations = [{'learning_rate': 0.001 * index} for index in range(1, 21)]
|
|
78
81
|
stream.initialize(configurations, max_steps=10, seed=42)
|
|
79
82
|
suggestion = stream.suggest()
|
|
80
83
|
# Train the suggested configuration locally, then report validation loss:
|
|
@@ -104,7 +107,7 @@ original target units. Lower is better. Test data is used only for final reporti
|
|
|
104
107
|
| --- | ---: | ---: | --- |
|
|
105
108
|
| `iris` | 10 | 100 | scikit-learn |
|
|
106
109
|
| `wine` | 200 | 2,000 | scikit-learn |
|
|
107
|
-
| `breast-cancer` |
|
|
110
|
+
| `breast-cancer` | 60 | 600 | scikit-learn |
|
|
108
111
|
| `digits` | 60 | 600 | scikit-learn |
|
|
109
112
|
| `glass` | 100 | 1,000 | OpenML dataset 41 |
|
|
110
113
|
| `vehicle` | 60 | 600 | OpenML dataset 54 |
|
|
@@ -120,8 +123,10 @@ grouped into one split. Preprocessing is fitted on training data only.
|
|
|
120
123
|
|
|
121
124
|
`--block-steps N` sets updates per observation and a maximum of `10*N` updates.
|
|
122
125
|
The defaults are bounded workload profiles, not guarantees of convergence.
|
|
123
|
-
Version
|
|
124
|
-
|
|
126
|
+
Version 3 raises the breast-cancer cap from 300 to 600 updates (60 per
|
|
127
|
+
observation); compare new breast-cancer results only with runs using that cap.
|
|
128
|
+
Version 2 introduced loss-based classification ranks, which are not comparable
|
|
129
|
+
with version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
|
|
125
130
|
`--split-seed`, and `--training-seed` allow independent overrides.
|
|
126
131
|
|
|
127
132
|
## Repeated benchmarks
|
|
@@ -185,3 +190,69 @@ Original SDK and demo code is licensed under the [MIT License](LICENSE).
|
|
|
185
190
|
The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
|
|
186
191
|
and FLAML (MIT). Their license texts and attribution are included in the package;
|
|
187
192
|
see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
|
|
193
|
+
|
|
194
|
+
## Optional MLflow tracking
|
|
195
|
+
|
|
196
|
+
Install the dashboard dependencies with the demo:
|
|
197
|
+
|
|
198
|
+
```sh
|
|
199
|
+
python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
|
|
200
|
+
python -m tuning_demo --api-url https://hyperopt.dev --dataset breast-cancer
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
In an interactive terminal, the service demo asks permission to start a local
|
|
204
|
+
dashboard. Answering y starts it on 127.0.0.1 and enables tracking. Pressing
|
|
205
|
+
Enter declines. For scripts, use `--mlflow-local` to enable local startup or
|
|
206
|
+
`--no-mlflow` to disable tracking and the prompt. An existing tracking server
|
|
207
|
+
can be selected explicitly with `--mlflow-tracking-uri URL`. These options
|
|
208
|
+
are mutually exclusive. `--mlflow-experiment` names the experiment.
|
|
209
|
+
|
|
210
|
+
The SDK starts one background process, waits for readiness, and reuses that
|
|
211
|
+
instance on later invocations. It prefers local port 5001 and selects another
|
|
212
|
+
available port if needed. The server stays running after training so the printed
|
|
213
|
+
dashboard link continues working. Its SQLite database, artifacts, state and server log persist
|
|
214
|
+
under `~/.local/share/tuning-client/mlflow` on macOS/Linux, respecting
|
|
215
|
+
`XDG_DATA_HOME`. Windows uses `LOCALAPPDATA/tuning-client/mlflow`.
|
|
216
|
+
The local instance uses the same Python environment as the demo.
|
|
217
|
+
Install the extra before opting in. Python 3.11+ supports both packages.
|
|
218
|
+
|
|
219
|
+
Tracking logs the completed service result and prints one link to that tuning
|
|
220
|
+
job. The dashboard highlights the selected model, plots candidate validation
|
|
221
|
+
losses against training steps, and shows each candidate's training allocation.
|
|
222
|
+
Hyperparameters and other candidates expand on demand. Tracking stores candidate
|
|
223
|
+
configurations and scalar metrics in the selected tracking server.
|
|
224
|
+
|
|
225
|
+
The fork supplies the `mlflow` Python namespace. Install it in an environment
|
|
226
|
+
without upstream `mlflow`, `mlflow-skinny` or `mlflow-tracing`, which own
|
|
227
|
+
overlapping files. The SDK can also log to an explicitly configured upstream
|
|
228
|
+
tracking server. Tracking errors propagate, and the normal JSON result is
|
|
229
|
+
saved before result logging begins.
|
|
230
|
+
|
|
231
|
+
Each completed experiment gets a parent run and one child run per candidate.
|
|
232
|
+
The child tagged `tuning.best_run=true` is the binary's selection from
|
|
233
|
+
`service_at_budget.ranking`, without reranking after local completion.
|
|
234
|
+
`service_validation_error` records that frozen selection metric; lower is
|
|
235
|
+
better. `validation_error` records the original observed curve at optimizer
|
|
236
|
+
steps. `final_validation_error`, `train_error`, and `test_error` record the
|
|
237
|
+
separate fully trained evaluation once. The parent tag `tuning.final_best_idx`
|
|
238
|
+
preserves the fully trained validation winner, which can differ from the
|
|
239
|
+
binary's winner. All candidate runs have `tuning.run=true`.
|
|
240
|
+
|
|
241
|
+
For a custom SDK workflow, build the same result shape as the demo and log it:
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
from tuning_client import MLflowTracker, start_local_mlflow
|
|
245
|
+
|
|
246
|
+
# Invoke after your application has obtained the user's consent.
|
|
247
|
+
tracking_uri = start_local_mlflow()
|
|
248
|
+
tracker = MLflowTracker(tracking_uri, experiment_name="Tuning")
|
|
249
|
+
run_id = tracker.log_result(result)
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
`result` contains `models` (each with `idx`, `configuration`, `steps`, `error`,
|
|
253
|
+
and optional `train_error`/`test_error`), `history` (`idx`, `steps`, `error`),
|
|
254
|
+
`best_validation_idx`, and optionally `service_at_budget` with `ranking` entries
|
|
255
|
+
(`idx`, `error`) and `rounds`. Without a service ranking, the supplied
|
|
256
|
+
`best_validation_idx` is used. Reusing the same result object on the same
|
|
257
|
+
tracker does not log it twice; cross-process retry deduplication is not provided.
|
|
258
|
+
The tracker leaves fluent MLflow active runs untouched.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tuning-client"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT AND Apache-2.0"
|
|
@@ -16,6 +16,7 @@ dependencies = ["httpx>=0.27,<1", "websockets>=15,<18"]
|
|
|
16
16
|
Homepage = "https://hyperopt.dev"
|
|
17
17
|
|
|
18
18
|
[project.optional-dependencies]
|
|
19
|
+
mlflow = ["tuning-mlflow==3.18.0.post1", "filelock>=3.12,<5"]
|
|
19
20
|
demo = ["numpy>=1.26,<3", "scikit-learn>=1.5,<2", "torch>=2.5,<3"]
|
|
20
21
|
custom = ["pandas>=2.2,<3", "pyarrow>=18,<24"]
|
|
21
22
|
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Synchronous SDK: training frameworks can call it without an async event loop."""
|
|
2
|
+
from ._local_mlflow import start_local_mlflow
|
|
3
|
+
from .mlflow import MLflowTracker
|
|
4
|
+
from .sdk import TuningClient, ProtocolError, TuningSession
|
|
5
|
+
|
|
6
|
+
__all__ = ["TuningClient", "ProtocolError", "TuningSession", "MLflowTracker", "start_local_mlflow"]
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Manage the loopback tracking server started by the tuning client."""
|
|
2
|
+
|
|
3
|
+
import http.client
|
|
4
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import secrets
|
|
10
|
+
import signal
|
|
11
|
+
import socket
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_HOST = "127.0.0.1"
|
|
18
|
+
_PREFERRED_PORT = 5001
|
|
19
|
+
_INSTANCE_HEADER = "X-Tuning-Local-Instance"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _data_directory():
|
|
23
|
+
standard_directory = os.getenv("LOCALAPPDATA" if os.name == "nt" else "XDG_DATA_HOME")
|
|
24
|
+
data_home = Path(standard_directory) if standard_directory else Path.home() / ".local" / "share"
|
|
25
|
+
return data_home / "tuning-client" / "mlflow"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _choose_port():
|
|
29
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation:
|
|
30
|
+
try:
|
|
31
|
+
reservation.bind((_HOST, _PREFERRED_PORT))
|
|
32
|
+
except OSError:
|
|
33
|
+
reservation.bind((_HOST, 0))
|
|
34
|
+
return reservation.getsockname()[1]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _is_ready(port, instance):
|
|
38
|
+
connection = http.client.HTTPConnection(_HOST, port, timeout=0.5)
|
|
39
|
+
try:
|
|
40
|
+
connection.request("GET", "/health")
|
|
41
|
+
response = connection.getresponse()
|
|
42
|
+
return response.status == 200 and response.getheader(_INSTANCE_HEADER) == instance
|
|
43
|
+
except (OSError, http.client.HTTPException):
|
|
44
|
+
return False
|
|
45
|
+
finally:
|
|
46
|
+
connection.close()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read_state(path):
|
|
50
|
+
try:
|
|
51
|
+
state = json.loads(path.read_text(encoding="utf-8"))
|
|
52
|
+
if (type(state["port"]) is int and 1 <= state["port"] <= 65535
|
|
53
|
+
and isinstance(state["instance"], str) and state["instance"]
|
|
54
|
+
and type(state["pending_until"]) in (int, float)):
|
|
55
|
+
return state
|
|
56
|
+
except (FileNotFoundError, ValueError, TypeError, KeyError):
|
|
57
|
+
pass
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _write_state(path, state):
|
|
62
|
+
pending = path.with_suffix(".tmp")
|
|
63
|
+
pending.write_text(json.dumps(state), encoding="utf-8")
|
|
64
|
+
pending.replace(path)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _wait_until_ready(port, instance, *, deadline, process=None):
|
|
68
|
+
while True:
|
|
69
|
+
if _is_ready(port, instance):
|
|
70
|
+
return True
|
|
71
|
+
if (process is not None and process.poll() is not None) or time.monotonic() >= deadline:
|
|
72
|
+
return False
|
|
73
|
+
time.sleep(0.2)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _launch_server(directory, port, instance, log_path):
|
|
77
|
+
command = [
|
|
78
|
+
sys.executable, "-m", "mlflow", "server", "--host", _HOST, "--port", str(port),
|
|
79
|
+
"--workers", "1", "--backend-store-uri", "sqlite:///mlflow.db",
|
|
80
|
+
"--serve-artifacts", "--artifacts-destination", (directory / "artifacts").as_uri(),
|
|
81
|
+
"--uvicorn-opts", f"--header {_INSTANCE_HEADER}:{instance}",
|
|
82
|
+
]
|
|
83
|
+
env = {
|
|
84
|
+
name: value for name, value in os.environ.items()
|
|
85
|
+
if not name.upper().startswith(("MLFLOW_", "UVICORN_"))
|
|
86
|
+
}
|
|
87
|
+
env["MLFLOW_DISABLE_TELEMETRY"] = "true"
|
|
88
|
+
detach = (
|
|
89
|
+
{"creationflags": subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP}
|
|
90
|
+
if os.name == "nt" else {"start_new_session": True}
|
|
91
|
+
)
|
|
92
|
+
with log_path.open("ab") as log:
|
|
93
|
+
return subprocess.Popen(command, cwd=directory, env=env, stdin=subprocess.DEVNULL,
|
|
94
|
+
stdout=log, stderr=subprocess.STDOUT, **detach)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _stop_failed_start(process):
|
|
98
|
+
# The launcher created this process and its process group. Never signal a
|
|
99
|
+
# PID read from state, which may belong to an unrelated process after a restart.
|
|
100
|
+
try:
|
|
101
|
+
if os.name == "nt":
|
|
102
|
+
taskkill = Path(os.getenv("SystemRoot", r"C:\Windows")) / "System32" / "taskkill.exe"
|
|
103
|
+
result = subprocess.run(
|
|
104
|
+
[str(taskkill), "/T", "/F", "/PID", str(process.pid)],
|
|
105
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
106
|
+
timeout=5, check=False,
|
|
107
|
+
)
|
|
108
|
+
if result.returncode:
|
|
109
|
+
return False
|
|
110
|
+
else:
|
|
111
|
+
os.killpg(process.pid, signal.SIGTERM)
|
|
112
|
+
process.wait(timeout=5)
|
|
113
|
+
return True
|
|
114
|
+
except ProcessLookupError:
|
|
115
|
+
return True
|
|
116
|
+
except subprocess.TimeoutExpired:
|
|
117
|
+
if os.name != "nt":
|
|
118
|
+
try:
|
|
119
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
120
|
+
process.wait(timeout=5)
|
|
121
|
+
return True
|
|
122
|
+
except ProcessLookupError:
|
|
123
|
+
return True
|
|
124
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
125
|
+
return False
|
|
126
|
+
return False
|
|
127
|
+
except OSError:
|
|
128
|
+
return False
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def start_local_mlflow(*, data_dir=None, timeout=60.0):
|
|
132
|
+
"""Start or reuse the local Tuning MLflow server and return its tracking URI.
|
|
133
|
+
|
|
134
|
+
Requires `tuning-client[mlflow]`. The server listens on loopback and stays
|
|
135
|
+
available after this Python process exits. Its database, artifacts, and
|
|
136
|
+
startup log are kept under the user's data directory or the given `data_dir`.
|
|
137
|
+
"""
|
|
138
|
+
if not math.isfinite(timeout) or timeout <= 0:
|
|
139
|
+
raise ValueError("Local MLflow startup timeout must be finite and positive")
|
|
140
|
+
try:
|
|
141
|
+
fork_version = version("tuning-mlflow")
|
|
142
|
+
from filelock import FileLock
|
|
143
|
+
from mlflow.version import VERSION
|
|
144
|
+
except (PackageNotFoundError, ImportError) as exc:
|
|
145
|
+
raise ImportError("Install 'tuning-client[mlflow]' to start local Tuning MLflow") from exc
|
|
146
|
+
if VERSION != fork_version:
|
|
147
|
+
raise ImportError("The imported MLflow does not match tuning-mlflow. "
|
|
148
|
+
"Install 'tuning-client[mlflow]' in a clean environment")
|
|
149
|
+
|
|
150
|
+
directory = Path(data_dir).expanduser().resolve() if data_dir is not None else _data_directory().resolve()
|
|
151
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
152
|
+
state_path = directory / "server.json"
|
|
153
|
+
log_path = directory / "server.log"
|
|
154
|
+
with FileLock(str(directory / "startup.lock"), timeout=timeout + 1):
|
|
155
|
+
state = _read_state(state_path)
|
|
156
|
+
if state is not None:
|
|
157
|
+
port, instance = state["port"], state["instance"]
|
|
158
|
+
# A caller interrupted during startup may have left a server still initializing.
|
|
159
|
+
remaining = min(timeout, max(0, state["pending_until"] - time.time()))
|
|
160
|
+
if _wait_until_ready(port, instance, deadline=time.monotonic() + remaining):
|
|
161
|
+
if state["pending_until"]:
|
|
162
|
+
_write_state(state_path, {**state, "pending_until": 0})
|
|
163
|
+
return f"http://{_HOST}:{port}"
|
|
164
|
+
|
|
165
|
+
port = _choose_port()
|
|
166
|
+
instance = secrets.token_hex(16)
|
|
167
|
+
state = {"port": port, "instance": instance, "pending_until": time.time() + timeout}
|
|
168
|
+
_write_state(state_path, state)
|
|
169
|
+
try:
|
|
170
|
+
process = _launch_server(directory, port, instance, log_path)
|
|
171
|
+
except OSError as exc:
|
|
172
|
+
_write_state(state_path, {**state, "pending_until": 0})
|
|
173
|
+
raise RuntimeError(f"Could not launch local Tuning MLflow. Server log is at {log_path}") from exc
|
|
174
|
+
if not _wait_until_ready(port, instance, deadline=time.monotonic() + timeout, process=process):
|
|
175
|
+
status = process.poll()
|
|
176
|
+
pending_until = 0
|
|
177
|
+
if status is None:
|
|
178
|
+
reason = "did not become ready in time"
|
|
179
|
+
if not _stop_failed_start(process):
|
|
180
|
+
pending_until = time.time() + timeout
|
|
181
|
+
reason += " and could not be stopped"
|
|
182
|
+
else:
|
|
183
|
+
reason = f"exited with status {status}"
|
|
184
|
+
_write_state(state_path, {**state, "pending_until": pending_until})
|
|
185
|
+
raise RuntimeError(f"Local Tuning MLflow {reason}. See {log_path}")
|
|
186
|
+
_write_state(state_path, {**state, "pending_until": 0})
|
|
187
|
+
return f"http://{_HOST}:{port}"
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Optional client-side tracking; compatible with tuning-mlflow and upstream MLflow."""
|
|
2
|
+
import json
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class MLflowTracker:
|
|
7
|
+
"""Log a completed tuning result without changing model selection or training.
|
|
8
|
+
|
|
9
|
+
Uses the tracking client directly, leaving any fluent active run untouched.
|
|
10
|
+
Only parameters, scalar metrics, and selected metadata are sent; datasets,
|
|
11
|
+
credentials, model weights, and local result paths are never uploaded.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def __init__(self, tracking_uri=None, *, experiment_name="Tuning", client=None):
|
|
15
|
+
if client is None:
|
|
16
|
+
try:
|
|
17
|
+
from mlflow.tracking import MlflowClient
|
|
18
|
+
except ImportError as exc:
|
|
19
|
+
raise ImportError("Install tuning-mlflow (or mlflow) to enable tracking") from exc
|
|
20
|
+
client = MlflowClient(tracking_uri=tracking_uri)
|
|
21
|
+
self.client = client
|
|
22
|
+
experiment = client.get_experiment_by_name(experiment_name)
|
|
23
|
+
self.experiment_id = (experiment.experiment_id if experiment is not None
|
|
24
|
+
else client.create_experiment(experiment_name))
|
|
25
|
+
self._logged = {}
|
|
26
|
+
|
|
27
|
+
def log_result(self, result):
|
|
28
|
+
"""Return a parent run ID; repeated calls on this result are idempotent.
|
|
29
|
+
|
|
30
|
+
The result follows the demo schema: models have idx, configuration,
|
|
31
|
+
steps, error, test_error; history contains idx, steps, error. The caller
|
|
32
|
+
supplies service_at_budget.ranking and best_validation_idx. The service
|
|
33
|
+
winner is preserved separately from the final fully trained winner.
|
|
34
|
+
Failures propagate; a partially written run is marked FAILED.
|
|
35
|
+
"""
|
|
36
|
+
# Keep a reference so Python cannot recycle the identity for another result.
|
|
37
|
+
previous = self._logged.get(id(result))
|
|
38
|
+
if previous is not None:
|
|
39
|
+
return previous[1]
|
|
40
|
+
models = result["models"]
|
|
41
|
+
ranking = result.get("service_at_budget", {}).get("ranking", [])
|
|
42
|
+
winner = ranking[0]["idx"] if ranking else result["best_validation_idx"]
|
|
43
|
+
service_errors = {row["idx"]: row["error"] for row in ranking}
|
|
44
|
+
selection_metric = "service_validation_error" if ranking else "final_validation_error"
|
|
45
|
+
if len({row["idx"] for row in models}) != len(models):
|
|
46
|
+
raise ValueError("Model indices must be unique")
|
|
47
|
+
if winner not in {row["idx"] for row in models}:
|
|
48
|
+
raise ValueError("Selected winner must identify a model")
|
|
49
|
+
for row in [*models, *result.get("history", [])]:
|
|
50
|
+
if not math.isfinite(float(row["error"])):
|
|
51
|
+
raise ValueError("Tracking errors must be finite")
|
|
52
|
+
client = self.client
|
|
53
|
+
parent = client.create_run(self.experiment_id, tags={
|
|
54
|
+
"mlflow.runName": str(result.get("dataset", "Tuning")),
|
|
55
|
+
"tuning.experiment": "true", "tuning.best_idx": str(winner),
|
|
56
|
+
"tuning.metric_direction": "minimize",
|
|
57
|
+
"tuning.final_best_idx": str(result["best_validation_idx"]),
|
|
58
|
+
"tuning.selection_metric": selection_metric,
|
|
59
|
+
}).info.run_id
|
|
60
|
+
opened = [parent]
|
|
61
|
+
try:
|
|
62
|
+
for key in ("dataset", "seed", "metric", "session_id", "selection_policy"):
|
|
63
|
+
if result.get(key) is not None:
|
|
64
|
+
client.log_param(parent, key, str(result[key]))
|
|
65
|
+
for row in models:
|
|
66
|
+
idx = row["idx"]
|
|
67
|
+
run = client.create_run(self.experiment_id, tags={
|
|
68
|
+
"mlflow.runName": f"Candidate {idx}", "mlflow.parentRunId": parent,
|
|
69
|
+
"tuning.run": "true", "tuning.best_run": str(idx == winner).lower(),
|
|
70
|
+
"tuning.metric_direction": "minimize",
|
|
71
|
+
"tuning.selection_metric": selection_metric,
|
|
72
|
+
"tuning.summary": f"Candidate {idx}: final validation error {row['error']:.6g}",
|
|
73
|
+
}).info.run_id
|
|
74
|
+
opened.append(run)
|
|
75
|
+
client.log_param(run, "candidate_idx", str(idx))
|
|
76
|
+
for key, value in row.get("configuration", {}).items():
|
|
77
|
+
encoded = (json.dumps(value, sort_keys=True) if isinstance(value, (dict, list))
|
|
78
|
+
else str(value))
|
|
79
|
+
client.log_param(run, str(key), encoded)
|
|
80
|
+
for observation in result.get("history", []):
|
|
81
|
+
if observation["idx"] == idx:
|
|
82
|
+
client.log_metric(run, "validation_error", float(observation["error"]),
|
|
83
|
+
step=int(observation["steps"]))
|
|
84
|
+
if idx in service_errors:
|
|
85
|
+
client.log_metric(run, "service_validation_error", float(service_errors[idx]),
|
|
86
|
+
step=int(result["service_at_budget"].get("rounds", 0)))
|
|
87
|
+
for source, metric in (("error", "final_validation_error"),
|
|
88
|
+
("test_error", "test_error"), ("train_error", "train_error")):
|
|
89
|
+
if source in row and math.isfinite(float(row[source])):
|
|
90
|
+
client.log_metric(run, metric, float(row[source]), step=int(row["steps"]))
|
|
91
|
+
client.set_terminated(run, status="FINISHED")
|
|
92
|
+
opened.remove(run)
|
|
93
|
+
client.set_terminated(parent, status="FINISHED")
|
|
94
|
+
opened.remove(parent)
|
|
95
|
+
except BaseException:
|
|
96
|
+
for run in reversed(opened):
|
|
97
|
+
try:
|
|
98
|
+
client.set_terminated(run, status="FAILED")
|
|
99
|
+
except Exception:
|
|
100
|
+
pass
|
|
101
|
+
raise
|
|
102
|
+
self._logged[id(result)] = (result, parent)
|
|
103
|
+
return parent
|
|
@@ -4,10 +4,12 @@ import getpass
|
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
6
|
import ssl
|
|
7
|
+
import sys
|
|
7
8
|
import time
|
|
8
9
|
from pathlib import Path
|
|
10
|
+
from urllib.parse import urlencode, urlsplit, urlunsplit
|
|
9
11
|
|
|
10
|
-
from tuning_client import TuningClient, TuningSession
|
|
12
|
+
from tuning_client import TuningClient, TuningSession, start_local_mlflow
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
def parser():
|
|
@@ -20,6 +22,7 @@ def parser():
|
|
|
20
22
|
p.add_argument("--target", help="Target column for --custom-data")
|
|
21
23
|
p.add_argument("--task", choices=("classification", "regression"), help="Required with --custom-data")
|
|
22
24
|
p.add_argument("--drop-columns", nargs="*", default=[], help="Exclude identifiers or leakage columns")
|
|
25
|
+
_add_mlflow_arguments(p)
|
|
23
26
|
p.add_argument("--no-telemetry", action="store_true", help="Disable optional workload usage events")
|
|
24
27
|
p.add_argument("--list-datasets", action="store_true")
|
|
25
28
|
p.add_argument("--interactive", action="store_true", help="Choose another dataset after each run")
|
|
@@ -47,6 +50,10 @@ def run_dataset(args, api, name, *, stream=None, reset=False):
|
|
|
47
50
|
if getattr(args, "scheduler", "service") != "service":
|
|
48
51
|
from .benchmarking.runner import run_dataset as run_local_dataset
|
|
49
52
|
return run_local_dataset(args, name)
|
|
53
|
+
tracker = None
|
|
54
|
+
if getattr(args, "mlflow_tracking_uri", None):
|
|
55
|
+
from tuning_client import MLflowTracker
|
|
56
|
+
tracker = MLflowTracker(args.mlflow_tracking_uri, experiment_name=args.mlflow_experiment)
|
|
50
57
|
from tuning_examples.telemetry import RunTelemetry
|
|
51
58
|
custom = getattr(args, "custom_data", None)
|
|
52
59
|
workload = ("custom_" + args.task if custom else
|
|
@@ -58,6 +65,27 @@ def run_dataset(args, api, name, *, stream=None, reset=False):
|
|
|
58
65
|
except BaseException:
|
|
59
66
|
telemetry.emit('workload_failed')
|
|
60
67
|
raise
|
|
68
|
+
if tracker is not None:
|
|
69
|
+
# MLflow normally prints two upstream UI links for every terminated run.
|
|
70
|
+
key = "MLFLOW_SUPPRESS_PRINTING_URL_TO_STDOUT"
|
|
71
|
+
previous = os.environ.get(key)
|
|
72
|
+
os.environ[key] = "true"
|
|
73
|
+
try:
|
|
74
|
+
run_id = tracker.log_result(result)
|
|
75
|
+
finally:
|
|
76
|
+
if previous is None:
|
|
77
|
+
os.environ.pop(key, None)
|
|
78
|
+
else:
|
|
79
|
+
os.environ[key] = previous
|
|
80
|
+
result["mlflow_run_id"] = run_id
|
|
81
|
+
uri = urlsplit(args.mlflow_tracking_uri)
|
|
82
|
+
if uri.scheme in ("http", "https") and uri.hostname:
|
|
83
|
+
query = urlencode({"experiment_id": tracker.experiment_id, "run_id": run_id})
|
|
84
|
+
host = uri.netloc.rsplit("@", 1)[-1]
|
|
85
|
+
dashboard = urlunsplit((uri.scheme, host, uri.path.rstrip("/") + "/", query, ""))
|
|
86
|
+
print(f"Tuning dashboard: {dashboard}")
|
|
87
|
+
else:
|
|
88
|
+
print(f"MLflow run: {run_id}")
|
|
61
89
|
telemetry.emit('workload_completed')
|
|
62
90
|
return result
|
|
63
91
|
|
|
@@ -178,6 +206,42 @@ def _run_dataset(args, api, name, *, stream=None, reset=False, telemetry):
|
|
|
178
206
|
return result
|
|
179
207
|
|
|
180
208
|
|
|
209
|
+
def _add_mlflow_arguments(p):
|
|
210
|
+
tracking = p.add_mutually_exclusive_group()
|
|
211
|
+
tracking.add_argument("--mlflow-tracking-uri", help="Opt in to client-side MLflow logging at this URI")
|
|
212
|
+
tracking.add_argument("--mlflow-local", action="store_true", help="Start or reuse a local Tuning MLflow dashboard")
|
|
213
|
+
tracking.add_argument("--no-mlflow", action="store_true", help="Disable MLflow logging and skip the prompt")
|
|
214
|
+
p.add_argument("--mlflow-experiment", default="Tuning", help="MLflow experiment name")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _configure_mlflow(p, args):
|
|
218
|
+
if args.scheduler != "service":
|
|
219
|
+
if args.mlflow_tracking_uri or args.mlflow_local:
|
|
220
|
+
p.error("MLflow logging currently supports the service scheduler")
|
|
221
|
+
return
|
|
222
|
+
if args.mlflow_tracking_uri:
|
|
223
|
+
try:
|
|
224
|
+
from mlflow.tracking import MlflowClient
|
|
225
|
+
except ImportError as exc:
|
|
226
|
+
p.error(f"Install 'tuning-client[mlflow]' or 'mlflow' to enable tracking ({exc})")
|
|
227
|
+
return
|
|
228
|
+
if args.no_mlflow:
|
|
229
|
+
return
|
|
230
|
+
use_local = args.mlflow_local
|
|
231
|
+
if not use_local and sys.stdin is not None and sys.stdin.isatty():
|
|
232
|
+
try:
|
|
233
|
+
answer = input("Start a local Tuning MLflow dashboard for this run? [y/N] ")
|
|
234
|
+
except EOFError:
|
|
235
|
+
return
|
|
236
|
+
use_local = answer.strip().lower() in ("y", "yes")
|
|
237
|
+
if use_local:
|
|
238
|
+
try:
|
|
239
|
+
args.mlflow_tracking_uri = start_local_mlflow()
|
|
240
|
+
except (ImportError, RuntimeError, OSError) as exc:
|
|
241
|
+
p.error(str(exc))
|
|
242
|
+
print(f"Local Tuning MLflow: {args.mlflow_tracking_uri}", flush=True)
|
|
243
|
+
|
|
244
|
+
|
|
181
245
|
def main():
|
|
182
246
|
p = parser()
|
|
183
247
|
args = p.parse_args()
|
|
@@ -210,6 +274,7 @@ def main():
|
|
|
210
274
|
p.error("interactive runs create a fresh session per dataset; omit --session-id and --direct")
|
|
211
275
|
if (args.tls_ca_file or args.tls_server_name) and not args.stream_url:
|
|
212
276
|
p.error("TLS overrides require --stream-url; normal API mode discovers the certificate")
|
|
277
|
+
_configure_mlflow(p, args)
|
|
213
278
|
import torch
|
|
214
279
|
torch.set_num_threads(1)
|
|
215
280
|
with (nullcontext(None) if args.direct or local else TuningClient(
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
The ASHA, Median Stopping Rule, PBT, and PB2 implementations were derived from the Python
|
|
6
6
|
scheduler sources in Ray **2.59.0**, licensed under Apache-2.0.
|
|
7
|
-
The local benchmark runtime does not import Ray or start a Ray
|
|
7
|
+
The local benchmark runtime does not import Ray or start a Ray runtime.
|
|
8
8
|
|
|
9
9
|
Upstream sources:
|
|
10
10
|
|
|
@@ -17,7 +17,7 @@ Upstream sources:
|
|
|
17
17
|
`LICENSE-Ray.txt` contains the Apache-2.0 terms. The installed Ray wheel's license
|
|
18
18
|
file identifies the license as "Apache 2.0" without separate copyright notices.
|
|
19
19
|
|
|
20
|
-
Changes: removed Ray trials/controllers,
|
|
20
|
+
Changes: removed Ray trials/controllers, distributed execution, actor management,
|
|
21
21
|
logging, pandas bookkeeping, and disk checkpoints. ASHA uses one bracket and
|
|
22
22
|
minimizes validation error. Median Stopping Rule uses training-block progress,
|
|
23
23
|
best observed loss and the median of peer running-average losses; with too
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
"""Client training resources; the service still counts ten observations per model."""
|
|
2
2
|
from dataclasses import asdict, dataclass
|
|
3
3
|
|
|
4
|
-
BENCHMARK_VERSION =
|
|
4
|
+
BENCHMARK_VERSION = 3
|
|
5
5
|
# Calibrated on seeds 42/43 using top-three mean validation loss (5% tolerance).
|
|
6
|
+
# Breast-cancer revised to 600 updates after the 2026-11-15 independent
|
|
7
|
+
# 20-candidate, ten-seed audit; other profiles retain the original calibration.
|
|
6
8
|
# Ten observations per model.
|
|
7
9
|
BLOCK_STEPS = {
|
|
8
|
-
'iris': 10, 'wine': 200, 'breast-cancer':
|
|
10
|
+
'iris': 10, 'wine': 200, 'breast-cancer': 60, 'digits': 60,
|
|
9
11
|
'glass': 100, 'vehicle': 60, 'diabetes': 10, 'friedman': 200,
|
|
10
12
|
'linear-regression': 200, 'toy-lm': 30, 'toy-vla': 200,
|
|
11
13
|
}
|
|
@@ -15,7 +15,7 @@ from statistics import mean
|
|
|
15
15
|
import time
|
|
16
16
|
|
|
17
17
|
from tuning_client import TuningClient
|
|
18
|
-
from .__main__ import run_dataset
|
|
18
|
+
from .__main__ import _add_mlflow_arguments, _configure_mlflow, run_dataset
|
|
19
19
|
from .datasets import DATASETS
|
|
20
20
|
from .training import error_ranks
|
|
21
21
|
from .timing import assess_operations, print_assessment
|
|
@@ -252,6 +252,7 @@ def main():
|
|
|
252
252
|
description="Average final ranks over repeated tuning demos (one tuning session per dataset)")
|
|
253
253
|
from .benchmarking import add_arguments, validate_arguments
|
|
254
254
|
add_arguments(parser)
|
|
255
|
+
_add_mlflow_arguments(parser)
|
|
255
256
|
parser.add_argument("--api-url", default=os.getenv("COMPUTE_API_URL"))
|
|
256
257
|
parser.add_argument("--email", default=os.getenv("COMPUTE_EMAIL"))
|
|
257
258
|
parser.add_argument("--register", action="store_true", help="Register once before logging in")
|
|
@@ -295,6 +296,7 @@ def main():
|
|
|
295
296
|
if args.summary_file is None:
|
|
296
297
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
|
|
297
298
|
args.summary_file = args.output_dir / f"statistics-{args.scheduler}-{timestamp}.json"
|
|
299
|
+
_configure_mlflow(parser, args)
|
|
298
300
|
import torch
|
|
299
301
|
torch.set_num_threads(1)
|
|
300
302
|
count = 0 if local else 1 if args.session_scope == "run" else len(args.datasets)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|