tuning-client 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {tuning_client-0.2.0 → tuning_client-0.2.2}/PKG-INFO +83 -9
  2. {tuning_client-0.2.0 → tuning_client-0.2.2}/README.md +79 -8
  3. {tuning_client-0.2.0 → tuning_client-0.2.2}/pyproject.toml +2 -1
  4. tuning_client-0.2.2/tuning_client/__init__.py +6 -0
  5. tuning_client-0.2.2/tuning_client/_local_mlflow.py +187 -0
  6. tuning_client-0.2.2/tuning_client/mlflow.py +103 -0
  7. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_client.egg-info/SOURCES.txt +2 -0
  8. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/__main__.py +66 -1
  9. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/THIRD_PARTY.md +2 -2
  10. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/budgets.py +4 -2
  11. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/stats.py +3 -1
  12. tuning_client-0.2.0/tuning_client/__init__.py +0 -4
  13. {tuning_client-0.2.0 → tuning_client-0.2.2}/LICENSE +0 -0
  14. {tuning_client-0.2.0 → tuning_client-0.2.2}/MANIFEST.in +0 -0
  15. {tuning_client-0.2.0 → tuning_client-0.2.2}/setup.cfg +0 -0
  16. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_client/sdk.py +0 -0
  17. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/__init__.py +0 -0
  18. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/LICENSE-FLAML.txt +0 -0
  19. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/LICENSE-Ray.txt +0 -0
  20. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/README.md +0 -0
  21. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/__init__.py +0 -0
  22. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/asha.py +0 -0
  23. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/flaml.py +0 -0
  24. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/median.py +0 -0
  25. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/parallel.py +0 -0
  26. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/pb2.py +0 -0
  27. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/pb2_utils.py +0 -0
  28. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/pbt.py +0 -0
  29. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/population.py +0 -0
  30. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/benchmarking/runner.py +0 -0
  31. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/datasets.py +0 -0
  32. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/timing.py +0 -0
  33. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_demo/training.py +0 -0
  34. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/__init__.py +0 -0
  35. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/custom.py +0 -0
  36. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/telemetry.py +0 -0
  37. {tuning_client-0.2.0 → tuning_client-0.2.2}/tuning_examples/toys.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tuning-client
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service
5
5
  License-Expression: MIT AND Apache-2.0
6
6
  Project-URL: Homepage, https://hyperopt.dev
@@ -12,6 +12,9 @@ License-File: tuning_demo/benchmarking/LICENSE-Ray.txt
12
12
  License-File: tuning_demo/benchmarking/THIRD_PARTY.md
13
13
  Requires-Dist: httpx<1,>=0.27
14
14
  Requires-Dist: websockets<18,>=15
15
+ Provides-Extra: mlflow
16
+ Requires-Dist: tuning-mlflow==3.18.0.post1; extra == "mlflow"
17
+ Requires-Dist: filelock<5,>=3.12; extra == "mlflow"
15
18
  Provides-Extra: demo
16
19
  Requires-Dist: numpy<3,>=1.26; extra == "demo"
17
20
  Requires-Dist: scikit-learn<2,>=1.5; extra == "demo"
@@ -49,11 +52,11 @@ selected by the default PyPI distribution:
49
52
 
50
53
  ```sh
51
54
  python -m pip install 'torch>=2.5,<3' --index-url https://download.pytorch.org/whl/cpu
52
- python -m pip install 'tuning-client[demo]'
55
+ python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
53
56
  ```
54
57
 
55
58
  The installed CPU build satisfies the demo's PyTorch requirement. On macOS,
56
- install `tuning-client[demo]` directly. See the
59
+ install `tuning-client[demo,mlflow]>=0.2.2` directly. See the
57
60
  [PyTorch installation guide](https://pytorch.org/get-started/locally/) for other
58
61
  platforms. If a failed download filled the disk, `python -m pip cache purge`
59
62
  removes pip's cached downloads; `--no-cache-dir` avoids caching the next install.
@@ -71,6 +74,11 @@ tuning-client-demo --api-url https://hyperopt.dev --email you@example.com --data
71
74
  python -m tuning_demo --list-datasets
72
75
  ```
73
76
 
77
+ The service demo first offers to start a local MLflow dashboard. Answer y to enable
78
+ tracking, or press Enter to continue. Use `--mlflow-local` to enable it explicitly
79
+ or `--no-mlflow` to skip the prompt. Noninteractive commands require
80
+ `--mlflow-local` or `--mlflow-tracking-uri` to enable tracking.
81
+
74
82
  The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
75
83
  or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
76
84
  `--api-url` each time. Connections use HTTPS and verified TLS.
@@ -95,9 +103,7 @@ with TuningClient(os.environ['COMPUTE_API_URL'],
95
103
  try:
96
104
  stream = api.tuning_session(session_id)
97
105
  try:
98
- configurations = [{'learning_rate': rate} for rate in
99
- [0.001, 0.002, 0.003, 0.004, 0.005,
100
- 0.006, 0.007, 0.008, 0.009, 0.010]]
106
+ configurations = [{'learning_rate': 0.001 * index} for index in range(1, 21)]
101
107
  stream.initialize(configurations, max_steps=10, seed=42)
102
108
  suggestion = stream.suggest()
103
109
  # Train the suggested configuration locally, then report validation loss:
@@ -127,7 +133,7 @@ original target units. Lower is better. Test data is used only for final reporti
127
133
  | --- | ---: | ---: | --- |
128
134
  | `iris` | 10 | 100 | scikit-learn |
129
135
  | `wine` | 200 | 2,000 | scikit-learn |
130
- | `breast-cancer` | 30 | 300 | scikit-learn |
136
+ | `breast-cancer` | 60 | 600 | scikit-learn |
131
137
  | `digits` | 60 | 600 | scikit-learn |
132
138
  | `glass` | 100 | 1,000 | OpenML dataset 41 |
133
139
  | `vehicle` | 60 | 600 | OpenML dataset 54 |
@@ -143,8 +149,10 @@ grouped into one split. Preprocessing is fitted on training data only.
143
149
 
144
150
  `--block-steps N` sets updates per observation and a maximum of `10*N` updates.
145
151
  The defaults are bounded workload profiles, not guarantees of convergence.
146
- Version 2 uses loss-based classification ranks; these are not comparable with
147
- version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
152
+ Version 3 raises the breast-cancer cap from 300 to 600 updates (60 per
153
+ observation); compare new breast-cancer results only with runs using that cap.
154
+ Version 2 introduced loss-based classification ranks, which are not comparable
155
+ with version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
148
156
  `--split-seed`, and `--training-seed` allow independent overrides.
149
157
 
150
158
  ## Repeated benchmarks
@@ -208,3 +216,69 @@ Original SDK and demo code is licensed under the [MIT License](LICENSE).
208
216
  The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
209
217
  and FLAML (MIT). Their license texts and attribution are included in the package;
210
218
  see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
219
+
220
+ ## Optional MLflow tracking
221
+
222
+ Install the dashboard dependencies with the demo:
223
+
224
+ ```sh
225
+ python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
226
+ python -m tuning_demo --api-url https://hyperopt.dev --dataset breast-cancer
227
+ ```
228
+
229
+ In an interactive terminal, the service demo asks permission to start a local
230
+ dashboard. Answering y starts it on 127.0.0.1 and enables tracking. Pressing
231
+ Enter declines. For scripts, use `--mlflow-local` to enable local startup or
232
+ `--no-mlflow` to disable tracking and the prompt. An existing tracking server
233
+ can be selected explicitly with `--mlflow-tracking-uri URL`. These options
234
+ are mutually exclusive. `--mlflow-experiment` names the experiment.
235
+
236
+ The SDK starts one background process, waits for readiness, and reuses that
237
+ instance on later invocations. It prefers local port 5001 and selects another
238
+ available port if needed. The server stays running after training so the printed
239
+ dashboard link continues working. Its SQLite database, artifacts, state and server log persist
240
+ under `~/.local/share/tuning-client/mlflow` on macOS/Linux, respecting
241
+ `XDG_DATA_HOME`. Windows uses `LOCALAPPDATA/tuning-client/mlflow`.
242
+ The local instance uses the same Python environment as the demo.
243
+ Install the extra before opting in. Python 3.11+ supports both packages.
244
+
245
+ Tracking logs the completed service result and prints one link to that tuning
246
+ job. The dashboard highlights the selected model, plots candidate validation
247
+ losses against training steps, and shows each candidate's training allocation.
248
+ Hyperparameters and other candidates expand on demand. Tracking stores candidate
249
+ configurations and scalar metrics in the selected tracking server.
250
+
251
+ The fork supplies the `mlflow` Python namespace. Install it in an environment
252
+ without upstream `mlflow`, `mlflow-skinny` or `mlflow-tracing`, which own
253
+ overlapping files. The SDK can also log to an explicitly configured upstream
254
+ tracking server. Tracking errors propagate, and the normal JSON result is
255
+ saved before result logging begins.
256
+
257
+ Each completed experiment gets a parent run and one child run per candidate.
258
+ The child tagged `tuning.best_run=true` is the binary's selection from
259
+ `service_at_budget.ranking`, without reranking after local completion.
260
+ `service_validation_error` records that frozen selection metric; lower is
261
+ better. `validation_error` records the original observed curve at optimizer
262
+ steps. `final_validation_error`, `train_error`, and `test_error` record the
263
+ separate fully trained evaluation once. The parent tag `tuning.final_best_idx`
264
+ preserves the fully trained validation winner, which can differ from the
265
+ binary's winner. All candidate runs have `tuning.run=true`.
266
+
267
+ For a custom SDK workflow, build the same result shape as the demo and log it:
268
+
269
+ ```python
270
+ from tuning_client import MLflowTracker, start_local_mlflow
271
+
272
+ # Invoke after your application has obtained the user's consent.
273
+ tracking_uri = start_local_mlflow()
274
+ tracker = MLflowTracker(tracking_uri, experiment_name="Tuning")
275
+ run_id = tracker.log_result(result)
276
+ ```
277
+
278
+ `result` contains `models` (each with `idx`, `configuration`, `steps`, `error`,
279
+ and optional `train_error`/`test_error`), `history` (`idx`, `steps`, `error`),
280
+ `best_validation_idx`, and optionally `service_at_budget` with `ranking` entries
281
+ (`idx`, `error`) and `rounds`. Without a service ranking, the supplied
282
+ `best_validation_idx` is used. Reusing the same result object on the same
283
+ tracker does not log it twice; cross-process retry deduplication is not provided.
284
+ The tracker leaves fluent MLflow active runs untouched.
@@ -26,11 +26,11 @@ selected by the default PyPI distribution:
26
26
 
27
27
  ```sh
28
28
  python -m pip install 'torch>=2.5,<3' --index-url https://download.pytorch.org/whl/cpu
29
- python -m pip install 'tuning-client[demo]'
29
+ python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
30
30
  ```
31
31
 
32
32
  The installed CPU build satisfies the demo's PyTorch requirement. On macOS,
33
- install `tuning-client[demo]` directly. See the
33
+ install `tuning-client[demo,mlflow]>=0.2.2` directly. See the
34
34
  [PyTorch installation guide](https://pytorch.org/get-started/locally/) for other
35
35
  platforms. If a failed download filled the disk, `python -m pip cache purge`
36
36
  removes pip's cached downloads; `--no-cache-dir` avoids caching the next install.
@@ -48,6 +48,11 @@ tuning-client-demo --api-url https://hyperopt.dev --email you@example.com --data
48
48
  python -m tuning_demo --list-datasets
49
49
  ```
50
50
 
51
+ The service demo first offers to start a local MLflow dashboard. Answer y to enable
52
+ tracking, or press Enter to continue. Use `--mlflow-local` to enable it explicitly
53
+ or `--no-mlflow` to skip the prompt. Noninteractive commands require
54
+ `--mlflow-local` or `--mlflow-tracking-uri` to enable tracking.
55
+
51
56
  The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
52
57
  or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
53
58
  `--api-url` each time. Connections use HTTPS and verified TLS.
@@ -72,9 +77,7 @@ with TuningClient(os.environ['COMPUTE_API_URL'],
72
77
  try:
73
78
  stream = api.tuning_session(session_id)
74
79
  try:
75
- configurations = [{'learning_rate': rate} for rate in
76
- [0.001, 0.002, 0.003, 0.004, 0.005,
77
- 0.006, 0.007, 0.008, 0.009, 0.010]]
80
+ configurations = [{'learning_rate': 0.001 * index} for index in range(1, 21)]
78
81
  stream.initialize(configurations, max_steps=10, seed=42)
79
82
  suggestion = stream.suggest()
80
83
  # Train the suggested configuration locally, then report validation loss:
@@ -104,7 +107,7 @@ original target units. Lower is better. Test data is used only for final reporti
104
107
  | --- | ---: | ---: | --- |
105
108
  | `iris` | 10 | 100 | scikit-learn |
106
109
  | `wine` | 200 | 2,000 | scikit-learn |
107
- | `breast-cancer` | 30 | 300 | scikit-learn |
110
+ | `breast-cancer` | 60 | 600 | scikit-learn |
108
111
  | `digits` | 60 | 600 | scikit-learn |
109
112
  | `glass` | 100 | 1,000 | OpenML dataset 41 |
110
113
  | `vehicle` | 60 | 600 | OpenML dataset 54 |
@@ -120,8 +123,10 @@ grouped into one split. Preprocessing is fitted on training data only.
120
123
 
121
124
  `--block-steps N` sets updates per observation and a maximum of `10*N` updates.
122
125
  The defaults are bounded workload profiles, not guarantees of convergence.
123
- Version 2 uses loss-based classification ranks; these are not comparable with
124
- version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
126
+ Version 3 raises the breast-cancer cap from 300 to 600 updates (60 per
127
+ observation); compare new breast-cancer results only with runs using that cap.
128
+ Version 2 introduced loss-based classification ranks, which are not comparable
129
+ with version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
125
130
  `--split-seed`, and `--training-seed` allow independent overrides.
126
131
 
127
132
  ## Repeated benchmarks
@@ -185,3 +190,69 @@ Original SDK and demo code is licensed under the [MIT License](LICENSE).
185
190
  The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
186
191
  and FLAML (MIT). Their license texts and attribution are included in the package;
187
192
  see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
193
+
194
+ ## Optional MLflow tracking
195
+
196
+ Install the dashboard dependencies with the demo:
197
+
198
+ ```sh
199
+ python -m pip install 'tuning-client[demo,mlflow]>=0.2.2'
200
+ python -m tuning_demo --api-url https://hyperopt.dev --dataset breast-cancer
201
+ ```
202
+
203
+ In an interactive terminal, the service demo asks permission to start a local
204
+ dashboard. Answering y starts it on 127.0.0.1 and enables tracking. Pressing
205
+ Enter declines. For scripts, use `--mlflow-local` to enable local startup or
206
+ `--no-mlflow` to disable tracking and the prompt. An existing tracking server
207
+ can be selected explicitly with `--mlflow-tracking-uri URL`. These options
208
+ are mutually exclusive. `--mlflow-experiment` names the experiment.
209
+
210
+ The SDK starts one background process, waits for readiness, and reuses that
211
+ instance on later invocations. It prefers local port 5001 and selects another
212
+ available port if needed. The server stays running after training so the printed
213
+ dashboard link continues working. Its SQLite database, artifacts, state and server log persist
214
+ under `~/.local/share/tuning-client/mlflow` on macOS/Linux, respecting
215
+ `XDG_DATA_HOME`. Windows uses `LOCALAPPDATA/tuning-client/mlflow`.
216
+ The local instance uses the same Python environment as the demo.
217
+ Install the extra before opting in. Python 3.11+ supports both packages.
218
+
219
+ Tracking logs the completed service result and prints one link to that tuning
220
+ job. The dashboard highlights the selected model, plots candidate validation
221
+ losses against training steps, and shows each candidate's training allocation.
222
+ Hyperparameters and other candidates expand on demand. Tracking stores candidate
223
+ configurations and scalar metrics in the selected tracking server.
224
+
225
+ The fork supplies the `mlflow` Python namespace. Install it in an environment
226
+ without upstream `mlflow`, `mlflow-skinny` or `mlflow-tracing`, which own
227
+ overlapping files. The SDK can also log to an explicitly configured upstream
228
+ tracking server. Tracking errors propagate, and the normal JSON result is
229
+ saved before result logging begins.
230
+
231
+ Each completed experiment gets a parent run and one child run per candidate.
232
+ The child tagged `tuning.best_run=true` is the binary's selection from
233
+ `service_at_budget.ranking`, without reranking after local completion.
234
+ `service_validation_error` records that frozen selection metric; lower is
235
+ better. `validation_error` records the original observed curve at optimizer
236
+ steps. `final_validation_error`, `train_error`, and `test_error` record the
237
+ separate fully trained evaluation once. The parent tag `tuning.final_best_idx`
238
+ preserves the fully trained validation winner, which can differ from the
239
+ binary's winner. All candidate runs have `tuning.run=true`.
240
+
241
+ For a custom SDK workflow, build the same result shape as the demo and log it:
242
+
243
+ ```python
244
+ from tuning_client import MLflowTracker, start_local_mlflow
245
+
246
+ # Invoke after your application has obtained the user's consent.
247
+ tracking_uri = start_local_mlflow()
248
+ tracker = MLflowTracker(tracking_uri, experiment_name="Tuning")
249
+ run_id = tracker.log_result(result)
250
+ ```
251
+
252
+ `result` contains `models` (each with `idx`, `configuration`, `steps`, `error`,
253
+ and optional `train_error`/`test_error`), `history` (`idx`, `steps`, `error`),
254
+ `best_validation_idx`, and optionally `service_at_budget` with `ranking` entries
255
+ (`idx`, `error`) and `rounds`. Without a service ranking, the supplied
256
+ `best_validation_idx` is used. Reusing the same result object on the same
257
+ tracker does not log it twice; cross-process retry deduplication is not provided.
258
+ The tracker leaves fluent MLflow active runs untouched.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tuning-client"
7
- version = "0.2.0"
7
+ version = "0.2.2"
8
8
  description = "Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service"
9
9
  readme = "README.md"
10
10
  license = "MIT AND Apache-2.0"
@@ -16,6 +16,7 @@ dependencies = ["httpx>=0.27,<1", "websockets>=15,<18"]
16
16
  Homepage = "https://hyperopt.dev"
17
17
 
18
18
  [project.optional-dependencies]
19
+ mlflow = ["tuning-mlflow==3.18.0.post1", "filelock>=3.12,<5"]
19
20
  demo = ["numpy>=1.26,<3", "scikit-learn>=1.5,<2", "torch>=2.5,<3"]
20
21
  custom = ["pandas>=2.2,<3", "pyarrow>=18,<24"]
21
22
 
@@ -0,0 +1,6 @@
1
+ """Synchronous SDK: training frameworks can call it without an async event loop."""
2
+ from ._local_mlflow import start_local_mlflow
3
+ from .mlflow import MLflowTracker
4
+ from .sdk import TuningClient, ProtocolError, TuningSession
5
+
6
+ __all__ = ["TuningClient", "ProtocolError", "TuningSession", "MLflowTracker", "start_local_mlflow"]
@@ -0,0 +1,187 @@
1
+ """Manage the loopback tracking server started by the tuning client."""
2
+
3
+ import http.client
4
+ from importlib.metadata import PackageNotFoundError, version
5
+ import json
6
+ import math
7
+ import os
8
+ from pathlib import Path
9
+ import secrets
10
+ import signal
11
+ import socket
12
+ import subprocess
13
+ import sys
14
+ import time
15
+
16
+
17
+ _HOST = "127.0.0.1"
18
+ _PREFERRED_PORT = 5001
19
+ _INSTANCE_HEADER = "X-Tuning-Local-Instance"
20
+
21
+
22
+ def _data_directory():
23
+ standard_directory = os.getenv("LOCALAPPDATA" if os.name == "nt" else "XDG_DATA_HOME")
24
+ data_home = Path(standard_directory) if standard_directory else Path.home() / ".local" / "share"
25
+ return data_home / "tuning-client" / "mlflow"
26
+
27
+
28
+ def _choose_port():
29
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as reservation:
30
+ try:
31
+ reservation.bind((_HOST, _PREFERRED_PORT))
32
+ except OSError:
33
+ reservation.bind((_HOST, 0))
34
+ return reservation.getsockname()[1]
35
+
36
+
37
+ def _is_ready(port, instance):
38
+ connection = http.client.HTTPConnection(_HOST, port, timeout=0.5)
39
+ try:
40
+ connection.request("GET", "/health")
41
+ response = connection.getresponse()
42
+ return response.status == 200 and response.getheader(_INSTANCE_HEADER) == instance
43
+ except (OSError, http.client.HTTPException):
44
+ return False
45
+ finally:
46
+ connection.close()
47
+
48
+
49
+ def _read_state(path):
50
+ try:
51
+ state = json.loads(path.read_text(encoding="utf-8"))
52
+ if (type(state["port"]) is int and 1 <= state["port"] <= 65535
53
+ and isinstance(state["instance"], str) and state["instance"]
54
+ and type(state["pending_until"]) in (int, float)):
55
+ return state
56
+ except (FileNotFoundError, ValueError, TypeError, KeyError):
57
+ pass
58
+ return None
59
+
60
+
61
+ def _write_state(path, state):
62
+ pending = path.with_suffix(".tmp")
63
+ pending.write_text(json.dumps(state), encoding="utf-8")
64
+ pending.replace(path)
65
+
66
+
67
+ def _wait_until_ready(port, instance, *, deadline, process=None):
68
+ while True:
69
+ if _is_ready(port, instance):
70
+ return True
71
+ if (process is not None and process.poll() is not None) or time.monotonic() >= deadline:
72
+ return False
73
+ time.sleep(0.2)
74
+
75
+
76
+ def _launch_server(directory, port, instance, log_path):
77
+ command = [
78
+ sys.executable, "-m", "mlflow", "server", "--host", _HOST, "--port", str(port),
79
+ "--workers", "1", "--backend-store-uri", "sqlite:///mlflow.db",
80
+ "--serve-artifacts", "--artifacts-destination", (directory / "artifacts").as_uri(),
81
+ "--uvicorn-opts", f"--header {_INSTANCE_HEADER}:{instance}",
82
+ ]
83
+ env = {
84
+ name: value for name, value in os.environ.items()
85
+ if not name.upper().startswith(("MLFLOW_", "UVICORN_"))
86
+ }
87
+ env["MLFLOW_DISABLE_TELEMETRY"] = "true"
88
+ detach = (
89
+ {"creationflags": subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP}
90
+ if os.name == "nt" else {"start_new_session": True}
91
+ )
92
+ with log_path.open("ab") as log:
93
+ return subprocess.Popen(command, cwd=directory, env=env, stdin=subprocess.DEVNULL,
94
+ stdout=log, stderr=subprocess.STDOUT, **detach)
95
+
96
+
97
+ def _stop_failed_start(process):
98
+ # The launcher created this process and its process group. Never signal a
99
+ # PID read from state, which may belong to an unrelated process after a restart.
100
+ try:
101
+ if os.name == "nt":
102
+ taskkill = Path(os.getenv("SystemRoot", r"C:\Windows")) / "System32" / "taskkill.exe"
103
+ result = subprocess.run(
104
+ [str(taskkill), "/T", "/F", "/PID", str(process.pid)],
105
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
106
+ timeout=5, check=False,
107
+ )
108
+ if result.returncode:
109
+ return False
110
+ else:
111
+ os.killpg(process.pid, signal.SIGTERM)
112
+ process.wait(timeout=5)
113
+ return True
114
+ except ProcessLookupError:
115
+ return True
116
+ except subprocess.TimeoutExpired:
117
+ if os.name != "nt":
118
+ try:
119
+ os.killpg(process.pid, signal.SIGKILL)
120
+ process.wait(timeout=5)
121
+ return True
122
+ except ProcessLookupError:
123
+ return True
124
+ except (OSError, subprocess.TimeoutExpired):
125
+ return False
126
+ return False
127
+ except OSError:
128
+ return False
129
+
130
+
131
+ def start_local_mlflow(*, data_dir=None, timeout=60.0):
132
+ """Start or reuse the local Tuning MLflow server and return its tracking URI.
133
+
134
+ Requires `tuning-client[mlflow]`. The server listens on loopback and stays
135
+ available after this Python process exits. Its database, artifacts, and
136
+ startup log are kept under the user's data directory or the given `data_dir`.
137
+ """
138
+ if not math.isfinite(timeout) or timeout <= 0:
139
+ raise ValueError("Local MLflow startup timeout must be finite and positive")
140
+ try:
141
+ fork_version = version("tuning-mlflow")
142
+ from filelock import FileLock
143
+ from mlflow.version import VERSION
144
+ except (PackageNotFoundError, ImportError) as exc:
145
+ raise ImportError("Install 'tuning-client[mlflow]' to start local Tuning MLflow") from exc
146
+ if VERSION != fork_version:
147
+ raise ImportError("The imported MLflow does not match tuning-mlflow. "
148
+ "Install 'tuning-client[mlflow]' in a clean environment")
149
+
150
+ directory = Path(data_dir).expanduser().resolve() if data_dir is not None else _data_directory().resolve()
151
+ directory.mkdir(mode=0o700, parents=True, exist_ok=True)
152
+ state_path = directory / "server.json"
153
+ log_path = directory / "server.log"
154
+ with FileLock(str(directory / "startup.lock"), timeout=timeout + 1):
155
+ state = _read_state(state_path)
156
+ if state is not None:
157
+ port, instance = state["port"], state["instance"]
158
+ # A caller interrupted during startup may have left a server still initializing.
159
+ remaining = min(timeout, max(0, state["pending_until"] - time.time()))
160
+ if _wait_until_ready(port, instance, deadline=time.monotonic() + remaining):
161
+ if state["pending_until"]:
162
+ _write_state(state_path, {**state, "pending_until": 0})
163
+ return f"http://{_HOST}:{port}"
164
+
165
+ port = _choose_port()
166
+ instance = secrets.token_hex(16)
167
+ state = {"port": port, "instance": instance, "pending_until": time.time() + timeout}
168
+ _write_state(state_path, state)
169
+ try:
170
+ process = _launch_server(directory, port, instance, log_path)
171
+ except OSError as exc:
172
+ _write_state(state_path, {**state, "pending_until": 0})
173
+ raise RuntimeError(f"Could not launch local Tuning MLflow. Server log is at {log_path}") from exc
174
+ if not _wait_until_ready(port, instance, deadline=time.monotonic() + timeout, process=process):
175
+ status = process.poll()
176
+ pending_until = 0
177
+ if status is None:
178
+ reason = "did not become ready in time"
179
+ if not _stop_failed_start(process):
180
+ pending_until = time.time() + timeout
181
+ reason += " and could not be stopped"
182
+ else:
183
+ reason = f"exited with status {status}"
184
+ _write_state(state_path, {**state, "pending_until": pending_until})
185
+ raise RuntimeError(f"Local Tuning MLflow {reason}. See {log_path}")
186
+ _write_state(state_path, {**state, "pending_until": 0})
187
+ return f"http://{_HOST}:{port}"
@@ -0,0 +1,103 @@
1
+ """Optional client-side tracking; compatible with tuning-mlflow and upstream MLflow."""
2
+ import json
3
+ import math
4
+
5
+
6
+ class MLflowTracker:
7
+ """Log a completed tuning result without changing model selection or training.
8
+
9
+ Uses the tracking client directly, leaving any fluent active run untouched.
10
+ Only parameters, scalar metrics, and selected metadata are sent; datasets,
11
+ credentials, model weights, and local result paths are never uploaded.
12
+ """
13
+
14
+ def __init__(self, tracking_uri=None, *, experiment_name="Tuning", client=None):
15
+ if client is None:
16
+ try:
17
+ from mlflow.tracking import MlflowClient
18
+ except ImportError as exc:
19
+ raise ImportError("Install tuning-mlflow (or mlflow) to enable tracking") from exc
20
+ client = MlflowClient(tracking_uri=tracking_uri)
21
+ self.client = client
22
+ experiment = client.get_experiment_by_name(experiment_name)
23
+ self.experiment_id = (experiment.experiment_id if experiment is not None
24
+ else client.create_experiment(experiment_name))
25
+ self._logged = {}
26
+
27
+ def log_result(self, result):
28
+ """Return a parent run ID; repeated calls on this result are idempotent.
29
+
30
+ The result follows the demo schema: models have idx, configuration,
31
+ steps, error, test_error; history contains idx, steps, error. The caller
32
+ supplies service_at_budget.ranking and best_validation_idx. The service
33
+ winner is preserved separately from the final fully trained winner.
34
+ Failures propagate; a partially written run is marked FAILED.
35
+ """
36
+ # Keep a reference so Python cannot recycle the identity for another result.
37
+ previous = self._logged.get(id(result))
38
+ if previous is not None:
39
+ return previous[1]
40
+ models = result["models"]
41
+ ranking = result.get("service_at_budget", {}).get("ranking", [])
42
+ winner = ranking[0]["idx"] if ranking else result["best_validation_idx"]
43
+ service_errors = {row["idx"]: row["error"] for row in ranking}
44
+ selection_metric = "service_validation_error" if ranking else "final_validation_error"
45
+ if len({row["idx"] for row in models}) != len(models):
46
+ raise ValueError("Model indices must be unique")
47
+ if winner not in {row["idx"] for row in models}:
48
+ raise ValueError("Selected winner must identify a model")
49
+ for row in [*models, *result.get("history", [])]:
50
+ if not math.isfinite(float(row["error"])):
51
+ raise ValueError("Tracking errors must be finite")
52
+ client = self.client
53
+ parent = client.create_run(self.experiment_id, tags={
54
+ "mlflow.runName": str(result.get("dataset", "Tuning")),
55
+ "tuning.experiment": "true", "tuning.best_idx": str(winner),
56
+ "tuning.metric_direction": "minimize",
57
+ "tuning.final_best_idx": str(result["best_validation_idx"]),
58
+ "tuning.selection_metric": selection_metric,
59
+ }).info.run_id
60
+ opened = [parent]
61
+ try:
62
+ for key in ("dataset", "seed", "metric", "session_id", "selection_policy"):
63
+ if result.get(key) is not None:
64
+ client.log_param(parent, key, str(result[key]))
65
+ for row in models:
66
+ idx = row["idx"]
67
+ run = client.create_run(self.experiment_id, tags={
68
+ "mlflow.runName": f"Candidate {idx}", "mlflow.parentRunId": parent,
69
+ "tuning.run": "true", "tuning.best_run": str(idx == winner).lower(),
70
+ "tuning.metric_direction": "minimize",
71
+ "tuning.selection_metric": selection_metric,
72
+ "tuning.summary": f"Candidate {idx}: final validation error {row['error']:.6g}",
73
+ }).info.run_id
74
+ opened.append(run)
75
+ client.log_param(run, "candidate_idx", str(idx))
76
+ for key, value in row.get("configuration", {}).items():
77
+ encoded = (json.dumps(value, sort_keys=True) if isinstance(value, (dict, list))
78
+ else str(value))
79
+ client.log_param(run, str(key), encoded)
80
+ for observation in result.get("history", []):
81
+ if observation["idx"] == idx:
82
+ client.log_metric(run, "validation_error", float(observation["error"]),
83
+ step=int(observation["steps"]))
84
+ if idx in service_errors:
85
+ client.log_metric(run, "service_validation_error", float(service_errors[idx]),
86
+ step=int(result["service_at_budget"].get("rounds", 0)))
87
+ for source, metric in (("error", "final_validation_error"),
88
+ ("test_error", "test_error"), ("train_error", "train_error")):
89
+ if source in row and math.isfinite(float(row[source])):
90
+ client.log_metric(run, metric, float(row[source]), step=int(row["steps"]))
91
+ client.set_terminated(run, status="FINISHED")
92
+ opened.remove(run)
93
+ client.set_terminated(parent, status="FINISHED")
94
+ opened.remove(parent)
95
+ except BaseException:
96
+ for run in reversed(opened):
97
+ try:
98
+ client.set_terminated(run, status="FAILED")
99
+ except Exception:
100
+ pass
101
+ raise
102
+ self._logged[id(result)] = (result, parent)
103
+ return parent
@@ -3,6 +3,8 @@ MANIFEST.in
3
3
  README.md
4
4
  pyproject.toml
5
5
  tuning_client/__init__.py
6
+ tuning_client/_local_mlflow.py
7
+ tuning_client/mlflow.py
6
8
  tuning_client/sdk.py
7
9
  tuning_demo/__init__.py
8
10
  tuning_demo/__main__.py
@@ -4,10 +4,12 @@ import getpass
4
4
  import json
5
5
  import os
6
6
  import ssl
7
+ import sys
7
8
  import time
8
9
  from pathlib import Path
10
+ from urllib.parse import urlencode, urlsplit, urlunsplit
9
11
 
10
- from tuning_client import TuningClient, TuningSession
12
+ from tuning_client import TuningClient, TuningSession, start_local_mlflow
11
13
 
12
14
 
13
15
  def parser():
@@ -20,6 +22,7 @@ def parser():
20
22
  p.add_argument("--target", help="Target column for --custom-data")
21
23
  p.add_argument("--task", choices=("classification", "regression"), help="Required with --custom-data")
22
24
  p.add_argument("--drop-columns", nargs="*", default=[], help="Exclude identifiers or leakage columns")
25
+ _add_mlflow_arguments(p)
23
26
  p.add_argument("--no-telemetry", action="store_true", help="Disable optional workload usage events")
24
27
  p.add_argument("--list-datasets", action="store_true")
25
28
  p.add_argument("--interactive", action="store_true", help="Choose another dataset after each run")
@@ -47,6 +50,10 @@ def run_dataset(args, api, name, *, stream=None, reset=False):
47
50
  if getattr(args, "scheduler", "service") != "service":
48
51
  from .benchmarking.runner import run_dataset as run_local_dataset
49
52
  return run_local_dataset(args, name)
53
+ tracker = None
54
+ if getattr(args, "mlflow_tracking_uri", None):
55
+ from tuning_client import MLflowTracker
56
+ tracker = MLflowTracker(args.mlflow_tracking_uri, experiment_name=args.mlflow_experiment)
50
57
  from tuning_examples.telemetry import RunTelemetry
51
58
  custom = getattr(args, "custom_data", None)
52
59
  workload = ("custom_" + args.task if custom else
@@ -58,6 +65,27 @@ def run_dataset(args, api, name, *, stream=None, reset=False):
58
65
  except BaseException:
59
66
  telemetry.emit('workload_failed')
60
67
  raise
68
+ if tracker is not None:
69
+ # MLflow normally prints two upstream UI links for every terminated run.
70
+ key = "MLFLOW_SUPPRESS_PRINTING_URL_TO_STDOUT"
71
+ previous = os.environ.get(key)
72
+ os.environ[key] = "true"
73
+ try:
74
+ run_id = tracker.log_result(result)
75
+ finally:
76
+ if previous is None:
77
+ os.environ.pop(key, None)
78
+ else:
79
+ os.environ[key] = previous
80
+ result["mlflow_run_id"] = run_id
81
+ uri = urlsplit(args.mlflow_tracking_uri)
82
+ if uri.scheme in ("http", "https") and uri.hostname:
83
+ query = urlencode({"experiment_id": tracker.experiment_id, "run_id": run_id})
84
+ host = uri.netloc.rsplit("@", 1)[-1]
85
+ dashboard = urlunsplit((uri.scheme, host, uri.path.rstrip("/") + "/", query, ""))
86
+ print(f"Tuning dashboard: {dashboard}")
87
+ else:
88
+ print(f"MLflow run: {run_id}")
61
89
  telemetry.emit('workload_completed')
62
90
  return result
63
91
 
@@ -178,6 +206,42 @@ def _run_dataset(args, api, name, *, stream=None, reset=False, telemetry):
178
206
  return result
179
207
 
180
208
 
209
+ def _add_mlflow_arguments(p):
210
+ tracking = p.add_mutually_exclusive_group()
211
+ tracking.add_argument("--mlflow-tracking-uri", help="Opt in to client-side MLflow logging at this URI")
212
+ tracking.add_argument("--mlflow-local", action="store_true", help="Start or reuse a local Tuning MLflow dashboard")
213
+ tracking.add_argument("--no-mlflow", action="store_true", help="Disable MLflow logging and skip the prompt")
214
+ p.add_argument("--mlflow-experiment", default="Tuning", help="MLflow experiment name")
215
+
216
+
217
+ def _configure_mlflow(p, args):
218
+ if args.scheduler != "service":
219
+ if args.mlflow_tracking_uri or args.mlflow_local:
220
+ p.error("MLflow logging currently supports the service scheduler")
221
+ return
222
+ if args.mlflow_tracking_uri:
223
+ try:
224
+ from mlflow.tracking import MlflowClient
225
+ except ImportError as exc:
226
+ p.error(f"Install 'tuning-client[mlflow]' or 'mlflow' to enable tracking ({exc})")
227
+ return
228
+ if args.no_mlflow:
229
+ return
230
+ use_local = args.mlflow_local
231
+ if not use_local and sys.stdin is not None and sys.stdin.isatty():
232
+ try:
233
+ answer = input("Start a local Tuning MLflow dashboard for this run? [y/N] ")
234
+ except EOFError:
235
+ return
236
+ use_local = answer.strip().lower() in ("y", "yes")
237
+ if use_local:
238
+ try:
239
+ args.mlflow_tracking_uri = start_local_mlflow()
240
+ except (ImportError, RuntimeError, OSError) as exc:
241
+ p.error(str(exc))
242
+ print(f"Local Tuning MLflow: {args.mlflow_tracking_uri}", flush=True)
243
+
244
+
181
245
  def main():
182
246
  p = parser()
183
247
  args = p.parse_args()
@@ -210,6 +274,7 @@ def main():
210
274
  p.error("interactive runs create a fresh session per dataset; omit --session-id and --direct")
211
275
  if (args.tls_ca_file or args.tls_server_name) and not args.stream_url:
212
276
  p.error("TLS overrides require --stream-url; normal API mode discovers the certificate")
277
+ _configure_mlflow(p, args)
213
278
  import torch
214
279
  torch.set_num_threads(1)
215
280
  with (nullcontext(None) if args.direct or local else TuningClient(
@@ -4,7 +4,7 @@
4
4
 
5
5
  The ASHA, Median Stopping Rule, PBT, and PB2 implementations were derived from the Python
6
6
  scheduler sources in Ray **2.59.0**, licensed under Apache-2.0.
7
- The local benchmark runtime does not import Ray or start a Ray cluster.
7
+ The local benchmark runtime does not import Ray or start a Ray runtime.
8
8
 
9
9
  Upstream sources:
10
10
 
@@ -17,7 +17,7 @@ Upstream sources:
17
17
  `LICENSE-Ray.txt` contains the Apache-2.0 terms. The installed Ray wheel's license
18
18
  file identifies the license as "Apache 2.0" without separate copyright notices.
19
19
 
20
- Changes: removed Ray trials/controllers, cluster execution, actor management,
20
+ Changes: removed Ray trials/controllers, distributed execution, actor management,
21
21
  logging, pandas bookkeeping, and disk checkpoints. ASHA uses one bracket and
22
22
  minimizes validation error. Median Stopping Rule uses training-block progress,
23
23
  best observed loss and the median of peer running-average losses; with too
@@ -1,11 +1,13 @@
1
1
  """Client training resources; the service still counts ten observations per model."""
2
2
  from dataclasses import asdict, dataclass
3
3
 
4
- BENCHMARK_VERSION = 2
4
+ BENCHMARK_VERSION = 3
5
5
  # Calibrated on seeds 42/43 using top-three mean validation loss (5% tolerance).
6
+ # Breast-cancer revised to 600 updates after the 2026-11-15 independent
7
+ # 20-candidate, ten-seed audit; other profiles retain the original calibration.
6
8
  # Ten observations per model.
7
9
  BLOCK_STEPS = {
8
- 'iris': 10, 'wine': 200, 'breast-cancer': 30, 'digits': 60,
10
+ 'iris': 10, 'wine': 200, 'breast-cancer': 60, 'digits': 60,
9
11
  'glass': 100, 'vehicle': 60, 'diabetes': 10, 'friedman': 200,
10
12
  'linear-regression': 200, 'toy-lm': 30, 'toy-vla': 200,
11
13
  }
@@ -15,7 +15,7 @@ from statistics import mean
15
15
  import time
16
16
 
17
17
  from tuning_client import TuningClient
18
- from .__main__ import run_dataset
18
+ from .__main__ import _add_mlflow_arguments, _configure_mlflow, run_dataset
19
19
  from .datasets import DATASETS
20
20
  from .training import error_ranks
21
21
  from .timing import assess_operations, print_assessment
@@ -252,6 +252,7 @@ def main():
252
252
  description="Average final ranks over repeated tuning demos (one tuning session per dataset)")
253
253
  from .benchmarking import add_arguments, validate_arguments
254
254
  add_arguments(parser)
255
+ _add_mlflow_arguments(parser)
255
256
  parser.add_argument("--api-url", default=os.getenv("COMPUTE_API_URL"))
256
257
  parser.add_argument("--email", default=os.getenv("COMPUTE_EMAIL"))
257
258
  parser.add_argument("--register", action="store_true", help="Register once before logging in")
@@ -295,6 +296,7 @@ def main():
295
296
  if args.summary_file is None:
296
297
  timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
297
298
  args.summary_file = args.output_dir / f"statistics-{args.scheduler}-{timestamp}.json"
299
+ _configure_mlflow(parser, args)
298
300
  import torch
299
301
  torch.set_num_threads(1)
300
302
  count = 0 if local else 1 if args.session_scope == "run" else len(args.datasets)
@@ -1,4 +0,0 @@
1
- """Synchronous SDK: training frameworks can call it without an async event loop."""
2
- from .sdk import TuningClient, ProtocolError, TuningSession
3
-
4
- __all__ = ["TuningClient", "ProtocolError", "TuningSession"]
File without changes
File without changes
File without changes