tuning-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tuning_client/__init__.py +4 -0
- tuning_client/sdk.py +344 -0
- tuning_client-0.1.0.dist-info/METADATA +199 -0
- tuning_client-0.1.0.dist-info/RECORD +36 -0
- tuning_client-0.1.0.dist-info/WHEEL +5 -0
- tuning_client-0.1.0.dist-info/entry_points.txt +2 -0
- tuning_client-0.1.0.dist-info/licenses/LICENSE +21 -0
- tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-FLAML.txt +21 -0
- tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-Ray.txt +177 -0
- tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/THIRD_PARTY.md +49 -0
- tuning_client-0.1.0.dist-info/top_level.txt +3 -0
- tuning_demo/__init__.py +1 -0
- tuning_demo/__main__.py +240 -0
- tuning_demo/benchmarking/LICENSE-FLAML.txt +21 -0
- tuning_demo/benchmarking/LICENSE-Ray.txt +177 -0
- tuning_demo/benchmarking/README.md +135 -0
- tuning_demo/benchmarking/THIRD_PARTY.md +49 -0
- tuning_demo/benchmarking/__init__.py +32 -0
- tuning_demo/benchmarking/asha.py +32 -0
- tuning_demo/benchmarking/flaml.py +70 -0
- tuning_demo/benchmarking/median.py +41 -0
- tuning_demo/benchmarking/parallel.py +107 -0
- tuning_demo/benchmarking/pb2.py +94 -0
- tuning_demo/benchmarking/pb2_utils.py +139 -0
- tuning_demo/benchmarking/pbt.py +60 -0
- tuning_demo/benchmarking/population.py +33 -0
- tuning_demo/benchmarking/runner.py +167 -0
- tuning_demo/budgets.py +38 -0
- tuning_demo/datasets.py +98 -0
- tuning_demo/stats.py +319 -0
- tuning_demo/timing.py +80 -0
- tuning_demo/training.py +239 -0
- tuning_examples/__init__.py +1 -0
- tuning_examples/custom.py +78 -0
- tuning_examples/telemetry.py +26 -0
- tuning_examples/toys.py +123 -0
tuning_client/sdk.py
ADDED
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import json
|
|
3
|
+
import ssl
|
|
4
|
+
import time
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
from websockets.exceptions import ConnectionClosed, InvalidHandshake
|
|
9
|
+
from websockets.sync.client import connect
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ProtocolError(RuntimeError):
|
|
13
|
+
"""The server rejected a request or cannot safely resume this session."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def validate_transport(url, secure_scheme, allow_insecure):
|
|
17
|
+
parts = urlsplit(url)
|
|
18
|
+
if (not parts.hostname or parts.username or parts.password or parts.query or parts.fragment):
|
|
19
|
+
raise ValueError("Transport URL must have a host and no credentials, query or fragment")
|
|
20
|
+
if parts.scheme == secure_scheme:
|
|
21
|
+
return
|
|
22
|
+
if (allow_insecure and parts.scheme == secure_scheme[:-1]
|
|
23
|
+
and parts.hostname in ("localhost", "127.0.0.1", "::1")):
|
|
24
|
+
return
|
|
25
|
+
raise ValueError(f"{secure_scheme.upper()} is required; allow_insecure=True permits loopback debugging only")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TuningClient:
|
|
29
|
+
def __init__(self, base_url, access_token=None, *, timeout=30, transport=None, allow_insecure=False):
|
|
30
|
+
validate_transport(base_url, "https", allow_insecure)
|
|
31
|
+
self.allow_insecure = allow_insecure
|
|
32
|
+
self.http = httpx.Client(base_url=base_url.rstrip("/"), timeout=timeout, transport=transport)
|
|
33
|
+
if access_token:
|
|
34
|
+
self.http.headers["Authorization"] = f"Bearer {access_token}"
|
|
35
|
+
|
|
36
|
+
def _request(self, method, path, **kwargs):
|
|
37
|
+
response = self.http.request(method, path, **kwargs)
|
|
38
|
+
try:
|
|
39
|
+
response.raise_for_status()
|
|
40
|
+
except httpx.HTTPStatusError as exc:
|
|
41
|
+
# FastAPI's human-readable reason is otherwise hidden by httpx.
|
|
42
|
+
# Do not include structured validation errors: those can echo input.
|
|
43
|
+
try:
|
|
44
|
+
payload = response.json()
|
|
45
|
+
except ValueError:
|
|
46
|
+
payload = None
|
|
47
|
+
detail = payload.get("detail") if isinstance(payload, dict) else None
|
|
48
|
+
if isinstance(detail, str) and detail:
|
|
49
|
+
raise httpx.HTTPStatusError(
|
|
50
|
+
f"{exc}\nAPI detail: {detail[:1000]}",
|
|
51
|
+
request=exc.request, response=exc.response,
|
|
52
|
+
) from None
|
|
53
|
+
raise
|
|
54
|
+
return response.json()
|
|
55
|
+
|
|
56
|
+
def register(self, email, password, *, challenge_token=""):
|
|
57
|
+
"""Public beta signup normally uses /signup in the browser."""
|
|
58
|
+
return self._request("POST", "/auth/register", json={"email": email, "password": password,
|
|
59
|
+
"challenge_token": challenge_token})
|
|
60
|
+
|
|
61
|
+
def verify_email(self, token):
|
|
62
|
+
return self._request("POST", "/auth/verify", json={"token": token})
|
|
63
|
+
|
|
64
|
+
def list_sessions(self):
|
|
65
|
+
return self._request("GET", "/sessions")
|
|
66
|
+
|
|
67
|
+
def wait_for_termination(self, session_id, *, timeout=180, poll_interval=2):
|
|
68
|
+
"""Wait until cleanup releases the user's single-session allowance."""
|
|
69
|
+
deadline = time.monotonic() + timeout
|
|
70
|
+
while any(s["session_id"] == session_id for s in self.list_sessions()):
|
|
71
|
+
remaining = deadline - time.monotonic()
|
|
72
|
+
if remaining <= 0:
|
|
73
|
+
raise TimeoutError(f"Cleanup pending for {session_id}; retry DELETE before creating another session")
|
|
74
|
+
time.sleep(min(poll_interval, remaining))
|
|
75
|
+
|
|
76
|
+
def login(self, email, password):
|
|
77
|
+
result = self._request("POST", "/auth/login", json={"email": email, "password": password})
|
|
78
|
+
self.http.headers["Authorization"] = f"Bearer {result['access_token']}"
|
|
79
|
+
|
|
80
|
+
def create_session(self):
|
|
81
|
+
# Never automatically retry creation: the API has no idempotency key.
|
|
82
|
+
return self._request("POST", "/sessions")["session_id"]
|
|
83
|
+
|
|
84
|
+
def terminate_session(self, session_id):
|
|
85
|
+
"""Delete the VM, unlike TuningSession.close(), which only disconnects."""
|
|
86
|
+
return self._request("DELETE", f"/sessions/{session_id}")
|
|
87
|
+
|
|
88
|
+
def connection_info(self, session_id):
|
|
89
|
+
return self._request("POST", f"/sessions/{session_id}/stream-token")
|
|
90
|
+
|
|
91
|
+
def stream_token(self, session_id):
|
|
92
|
+
return self.connection_info(session_id)["stream_token"]
|
|
93
|
+
|
|
94
|
+
def retry_session(self, session_id):
|
|
95
|
+
"""Retry/reconcile provisioning of this session's existing VM identity."""
|
|
96
|
+
return self._request("POST", f"/sessions/{session_id}/retry")
|
|
97
|
+
|
|
98
|
+
def wait_for_endpoint(self, session_id, *, timeout=600, poll_interval=2, on_progress=None):
|
|
99
|
+
"""Wait for an assigned secure endpoint; the API status is not a readiness signal."""
|
|
100
|
+
deadline = time.monotonic() + timeout
|
|
101
|
+
next_report = deadline - timeout
|
|
102
|
+
while True:
|
|
103
|
+
session = self.connection_info(session_id)
|
|
104
|
+
if session["status"] in ("STOPPING", "FINISHED"):
|
|
105
|
+
raise RuntimeError(f"Session {session_id} is {session['status']}")
|
|
106
|
+
if session.get("stream_url"):
|
|
107
|
+
validate_transport(session["stream_url"], "wss", self.allow_insecure)
|
|
108
|
+
return session
|
|
109
|
+
if session["status"] == "FAILED":
|
|
110
|
+
raise TimeoutError(f"Provisioning attempt failed for session {session_id}")
|
|
111
|
+
now = time.monotonic()
|
|
112
|
+
remaining = deadline - now
|
|
113
|
+
if on_progress and now >= next_report:
|
|
114
|
+
on_progress(f"Queued, position {session.get('queue_position', '?')}; cancel with terminate_session()"
|
|
115
|
+
if session.get('queued') else "Preparing your session")
|
|
116
|
+
next_report = now + 10
|
|
117
|
+
if remaining <= 0:
|
|
118
|
+
raise TimeoutError(f"Session {session_id} has no assigned endpoint yet")
|
|
119
|
+
time.sleep(min(poll_interval, remaining))
|
|
120
|
+
|
|
121
|
+
def wait_until_running(self, session_id, **kwargs):
|
|
122
|
+
"""Compatibility alias; use tuning_session() to check actual readiness."""
|
|
123
|
+
return self.wait_for_endpoint(session_id, **kwargs)
|
|
124
|
+
|
|
125
|
+
def tuning_session(self, session_id, *, stream_url=None, startup_timeout=600,
|
|
126
|
+
provisioning_retries=2, poll_interval=2, on_progress=None, **kwargs):
|
|
127
|
+
"""Connect directly to the VM, retrying provisioning of the SAME session.
|
|
128
|
+
|
|
129
|
+
startup_timeout bounds each attempt. No attempt creates another session.
|
|
130
|
+
An explicit endpoint override bypasses endpoint discovery.
|
|
131
|
+
"""
|
|
132
|
+
if startup_timeout <= 0 or provisioning_retries < 0 or poll_interval <= 0:
|
|
133
|
+
raise ValueError("Timeout/poll interval must be positive and retries non-negative")
|
|
134
|
+
def progress(message):
|
|
135
|
+
if on_progress:
|
|
136
|
+
on_progress(message)
|
|
137
|
+
|
|
138
|
+
network_errors = (OSError, TimeoutError, ConnectionClosed, InvalidHandshake, httpx.TransportError)
|
|
139
|
+
for attempt in range(provisioning_retries + 1):
|
|
140
|
+
session = None
|
|
141
|
+
deadline = time.monotonic() + startup_timeout
|
|
142
|
+
try:
|
|
143
|
+
tls_options = {}
|
|
144
|
+
if stream_url:
|
|
145
|
+
url = stream_url
|
|
146
|
+
else:
|
|
147
|
+
info = self.wait_for_endpoint(session_id, timeout=startup_timeout, poll_interval=poll_interval,
|
|
148
|
+
on_progress=on_progress)
|
|
149
|
+
progress("Session assigned")
|
|
150
|
+
url = info["stream_url"]
|
|
151
|
+
if info.get("tls_certificate"):
|
|
152
|
+
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
153
|
+
context.load_verify_locations(cadata=info["tls_certificate"])
|
|
154
|
+
tls_options = {"ssl_context": context, "tls_server_name": info["tls_server_name"]}
|
|
155
|
+
session = TuningSession(session_id, url, lambda: self.stream_token(session_id),
|
|
156
|
+
allow_insecure=self.allow_insecure, **tls_options, **kwargs)
|
|
157
|
+
connection_timeout = session.timeout
|
|
158
|
+
reported_error = None
|
|
159
|
+
progress("Connecting to session")
|
|
160
|
+
while True:
|
|
161
|
+
remaining = deadline - time.monotonic()
|
|
162
|
+
if remaining <= 0:
|
|
163
|
+
raise TimeoutError("VM stream did not become ready")
|
|
164
|
+
session.timeout = min(connection_timeout, remaining)
|
|
165
|
+
try:
|
|
166
|
+
session.connect()
|
|
167
|
+
session.timeout = connection_timeout
|
|
168
|
+
progress("Connected")
|
|
169
|
+
return session
|
|
170
|
+
except ssl.SSLCertVerificationError:
|
|
171
|
+
raise
|
|
172
|
+
except network_errors as exc:
|
|
173
|
+
error_type = type(exc).__name__
|
|
174
|
+
if reported_error != error_type:
|
|
175
|
+
progress("Waiting for connection")
|
|
176
|
+
reported_error = error_type
|
|
177
|
+
session.close()
|
|
178
|
+
time.sleep(min(poll_interval, max(0, deadline - time.monotonic())))
|
|
179
|
+
except ssl.SSLCertVerificationError:
|
|
180
|
+
raise
|
|
181
|
+
except network_errors:
|
|
182
|
+
if session:
|
|
183
|
+
session.close()
|
|
184
|
+
if attempt == provisioning_retries:
|
|
185
|
+
raise TimeoutError(
|
|
186
|
+
f"Session {session_id} did not become reachable after {attempt + 1} attempts; "
|
|
187
|
+
"retry this same session ID"
|
|
188
|
+
) from None
|
|
189
|
+
progress("Retrying session setup")
|
|
190
|
+
self.retry_session(session_id)
|
|
191
|
+
except Exception:
|
|
192
|
+
if session:
|
|
193
|
+
session.close()
|
|
194
|
+
raise
|
|
195
|
+
|
|
196
|
+
def close(self):
|
|
197
|
+
self.http.close()
|
|
198
|
+
|
|
199
|
+
def __enter__(self):
|
|
200
|
+
return self
|
|
201
|
+
|
|
202
|
+
def __exit__(self, *_):
|
|
203
|
+
self.close()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class TuningSession:
|
|
207
|
+
"""One sequential caller per session; reconnect state lives in this object.
|
|
208
|
+
|
|
209
|
+
token_provider is called on every new socket. A pending request is replayed
|
|
210
|
+
or resent with the SAME sequence; training must happen outside request().
|
|
211
|
+
Reuse this object after a network interruption. Process-restart recovery
|
|
212
|
+
would also require saving the client's model/optimizer/RNG state.
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
def __init__(self, session_id, stream_url, token_provider, *, timeout=30, retries=4, allow_insecure=False,
|
|
216
|
+
ssl_context=None, tls_server_name=None):
|
|
217
|
+
validate_transport(stream_url, "wss", allow_insecure)
|
|
218
|
+
if ssl_context is not None and (not ssl_context.check_hostname or ssl_context.verify_mode != ssl.CERT_REQUIRED):
|
|
219
|
+
raise ValueError("TLS certificate and hostname verification must remain enabled")
|
|
220
|
+
self.ssl_context = ssl_context
|
|
221
|
+
self.tls_server_name = tls_server_name
|
|
222
|
+
self.session_id = session_id
|
|
223
|
+
self.stream_url = stream_url
|
|
224
|
+
self.token_provider = token_provider
|
|
225
|
+
self.timeout = timeout
|
|
226
|
+
self.retries = retries
|
|
227
|
+
self.last_received_seq = 0
|
|
228
|
+
self.last_request_retried = False
|
|
229
|
+
self._socket = None
|
|
230
|
+
self._pending = None
|
|
231
|
+
self._replayed = None
|
|
232
|
+
|
|
233
|
+
def _receive(self):
|
|
234
|
+
message = json.loads(self._socket.recv(timeout=self.timeout))
|
|
235
|
+
if not isinstance(message, dict):
|
|
236
|
+
raise ProtocolError("Expected a JSON object")
|
|
237
|
+
if message.get("type") == "error":
|
|
238
|
+
if message.get("code") == "invalid_job":
|
|
239
|
+
self._pending = None # rejected jobs do not consume a sequence
|
|
240
|
+
raise ProtocolError(f"{message.get('code')}: {message.get('detail', '')}")
|
|
241
|
+
return message
|
|
242
|
+
|
|
243
|
+
def connect(self):
|
|
244
|
+
if self._socket is not None:
|
|
245
|
+
return self
|
|
246
|
+
try:
|
|
247
|
+
self._socket = connect(self.stream_url, open_timeout=self.timeout, close_timeout=2,
|
|
248
|
+
additional_headers={"Authorization": f"Bearer {self.token_provider()}"},
|
|
249
|
+
ssl=self.ssl_context, server_hostname=self.tls_server_name)
|
|
250
|
+
self._socket.send(json.dumps({"type": "resume", "session_id": self.session_id,
|
|
251
|
+
"last_received_seq": self.last_received_seq}))
|
|
252
|
+
response = self._receive()
|
|
253
|
+
if response.get("type") != "resume_ok" or response.get("session_id") != self.session_id:
|
|
254
|
+
raise ProtocolError("Invalid resume response")
|
|
255
|
+
server_seq = response.get("server_seq")
|
|
256
|
+
maximum = self.last_received_seq + (1 if self._pending else 0)
|
|
257
|
+
if type(server_seq) is not int or not self.last_received_seq <= server_seq <= maximum:
|
|
258
|
+
raise ProtocolError("Session state lost or another client advanced this session")
|
|
259
|
+
if server_seq > self.last_received_seq:
|
|
260
|
+
self._replayed = self._receive()
|
|
261
|
+
return self
|
|
262
|
+
except Exception:
|
|
263
|
+
self.close()
|
|
264
|
+
raise
|
|
265
|
+
|
|
266
|
+
def request(self, operation, **payload):
|
|
267
|
+
job = {"job": "tuning", "op": operation, **payload}
|
|
268
|
+
encoded = base64.b64encode(json.dumps(job, allow_nan=False).encode()).decode()
|
|
269
|
+
frame = {"type": "data", "seq": self.last_received_seq + 1, "data": encoded}
|
|
270
|
+
if self._pending is not None and self._pending != frame:
|
|
271
|
+
raise ProtocolError("Retry the pending request before submitting another operation")
|
|
272
|
+
self.last_request_retried = self._pending is not None
|
|
273
|
+
self._pending = frame
|
|
274
|
+
for attempt in range(self.retries + 1):
|
|
275
|
+
try:
|
|
276
|
+
self.connect()
|
|
277
|
+
if self._replayed is not None:
|
|
278
|
+
response, self._replayed = self._replayed, None
|
|
279
|
+
else:
|
|
280
|
+
self._socket.send(json.dumps(frame))
|
|
281
|
+
response = self._receive()
|
|
282
|
+
if response.get("type") != "result" or response.get("seq") != frame["seq"]:
|
|
283
|
+
raise ProtocolError("Unexpected result sequence")
|
|
284
|
+
result = json.loads(base64.b64decode(response["data"], validate=True))
|
|
285
|
+
if not isinstance(result, dict) or result.get("job") != "tuning" or result.get("op") != operation:
|
|
286
|
+
raise ProtocolError("Unexpected tuning result")
|
|
287
|
+
self.last_received_seq = frame["seq"]
|
|
288
|
+
self.last_request_retried |= bool(response.get("replayed", False))
|
|
289
|
+
self._pending = None
|
|
290
|
+
return result
|
|
291
|
+
except ssl.SSLCertVerificationError:
|
|
292
|
+
self.close()
|
|
293
|
+
raise
|
|
294
|
+
except (OSError, TimeoutError, ConnectionClosed, InvalidHandshake, httpx.TransportError):
|
|
295
|
+
self.last_request_retried = True
|
|
296
|
+
self.close()
|
|
297
|
+
if attempt == self.retries:
|
|
298
|
+
raise ConnectionError("Stream retries exhausted; retry the same operation on this object") from None
|
|
299
|
+
time.sleep(min(0.5 * 2**attempt, 5))
|
|
300
|
+
except Exception:
|
|
301
|
+
self.close()
|
|
302
|
+
raise
|
|
303
|
+
|
|
304
|
+
def initialize(self, configs, *, max_steps, seed=None, effort="medium"):
|
|
305
|
+
"""Budgets count observations, not client optimizer updates."""
|
|
306
|
+
return self.request("initialize", configs=configs,
|
|
307
|
+
max_steps=max_steps, seed=seed, effort=effort)
|
|
308
|
+
|
|
309
|
+
def suggest(self):
|
|
310
|
+
return self.request("suggest")
|
|
311
|
+
|
|
312
|
+
def reset(self, configs, *, max_steps, seed=None, effort="medium"):
|
|
313
|
+
"""Start fresh native state after completing the previous run's budget."""
|
|
314
|
+
return self.request("reset", configs=configs,
|
|
315
|
+
max_steps=max_steps, seed=seed, effort=effort)
|
|
316
|
+
|
|
317
|
+
def report(self, idx, error, steps=None):
|
|
318
|
+
"""Report one error; optional steps is the model's cumulative pull count."""
|
|
319
|
+
fields = {} if steps is None else {"steps": steps}
|
|
320
|
+
return self.request("report", idx=idx, error=error, **fields)
|
|
321
|
+
|
|
322
|
+
def finish(self, final_errors=None):
|
|
323
|
+
fields = {} if final_errors is None else {"final_errors": final_errors}
|
|
324
|
+
return self.request("finish", **fields)
|
|
325
|
+
|
|
326
|
+
def report_and_suggest(self, idx, error, steps=None):
|
|
327
|
+
"""Accept one observation and return the next suggestion in one reply."""
|
|
328
|
+
fields = {} if steps is None else {"steps": steps}
|
|
329
|
+
return self.request("report_and_suggest", idx=idx, error=error, **fields)
|
|
330
|
+
|
|
331
|
+
def status(self):
|
|
332
|
+
return self.request("status")
|
|
333
|
+
|
|
334
|
+
def close(self):
|
|
335
|
+
if self._socket is not None:
|
|
336
|
+
self._socket.close()
|
|
337
|
+
self._socket = None
|
|
338
|
+
self._replayed = None
|
|
339
|
+
|
|
340
|
+
def __enter__(self):
|
|
341
|
+
return self.connect()
|
|
342
|
+
|
|
343
|
+
def __exit__(self, *_):
|
|
344
|
+
self.close()
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tuning-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service
|
|
5
|
+
License-Expression: MIT AND Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://hyperopt.dev
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: tuning_demo/benchmarking/LICENSE-FLAML.txt
|
|
11
|
+
License-File: tuning_demo/benchmarking/LICENSE-Ray.txt
|
|
12
|
+
License-File: tuning_demo/benchmarking/THIRD_PARTY.md
|
|
13
|
+
Requires-Dist: httpx<1,>=0.27
|
|
14
|
+
Requires-Dist: websockets<18,>=15
|
|
15
|
+
Provides-Extra: demo
|
|
16
|
+
Requires-Dist: numpy<3,>=1.26; extra == "demo"
|
|
17
|
+
Requires-Dist: scikit-learn<2,>=1.5; extra == "demo"
|
|
18
|
+
Requires-Dist: torch<3,>=2.5; extra == "demo"
|
|
19
|
+
Provides-Extra: custom
|
|
20
|
+
Requires-Dist: pandas<3,>=2.2; extra == "custom"
|
|
21
|
+
Requires-Dist: pyarrow<24,>=18; extra == "custom"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# Tuning Client Python SDK and demo
|
|
25
|
+
|
|
26
|
+
Tuning Client is the Python SDK and demo for the
|
|
27
|
+
[hyperopt.dev AutoML service](https://hyperopt.dev).
|
|
28
|
+
|
|
29
|
+
Use the Python SDK to request tuning suggestions while keeping model training,
|
|
30
|
+
datasets, and model weights on your computer. The optional demo trains small
|
|
31
|
+
PyTorch models and compares tuning strategies on classification, regression,
|
|
32
|
+
and teaching datasets.
|
|
33
|
+
|
|
34
|
+
**Code provenance:** The SDK and demo code were generated with ChatGPT and
|
|
35
|
+
reviewed. Review does not guarantee correctness; evaluate the code and benchmark
|
|
36
|
+
results for your use case. The demos are examples, not production ML benchmarks.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
Requires Python 3.11 or newer. Install the SDK alone or include the demo:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
python -m pip install tuning-client
|
|
44
|
+
python -m pip install 'tuning-client[demo]'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Run the demo
|
|
48
|
+
|
|
49
|
+
Create an account using the service's signup page and verify your email. Replace
|
|
50
|
+
`https://YOUR_API` below with the service URL provided to you:
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
python -m tuning_demo --api-url https://YOUR_API --email you@example.com --dataset digits
|
|
54
|
+
# The installed command is equivalent:
|
|
55
|
+
tuning-client-demo --api-url https://YOUR_API --email you@example.com --dataset iris
|
|
56
|
+
python -m tuning_demo --list-datasets
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
|
|
60
|
+
or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
|
|
61
|
+
`--api-url` each time. Connections use HTTPS and verified TLS.
|
|
62
|
+
|
|
63
|
+
Service effort defaults to **medium**. Use `--effort easy` for lower effort;
|
|
64
|
+
the observation budget stays the same. `--interactive` offers another dataset
|
|
65
|
+
after each run. Sessions created by the demo are released automatically on
|
|
66
|
+
completion or failure. `--session-id ID` uses an existing unused session that
|
|
67
|
+
you remain responsible for releasing; `--keep-session` retains a new session.
|
|
68
|
+
|
|
69
|
+
Results are saved in `./results/`. Downloaded datasets are cached in
|
|
70
|
+
`~/.cache/tuning-client/`. Override these with `--output-dir` and `--data-dir`.
|
|
71
|
+
Optional usage events include a run ID, session ID, workload category, and
|
|
72
|
+
progress event; use `--no-telemetry` to disable them. Training data, file paths,
|
|
73
|
+
and model weights are not included in those events.
|
|
74
|
+
|
|
75
|
+
## SDK usage
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import os
|
|
79
|
+
from tuning_client import TuningClient
|
|
80
|
+
|
|
81
|
+
with TuningClient(os.environ['COMPUTE_API_URL'],
|
|
82
|
+
access_token=os.environ['COMPUTE_ACCESS_TOKEN']) as api:
|
|
83
|
+
session_id = api.create_session()
|
|
84
|
+
try:
|
|
85
|
+
stream = api.tuning_session(session_id)
|
|
86
|
+
try:
|
|
87
|
+
configurations = [{'learning_rate': rate} for rate in
|
|
88
|
+
[0.001, 0.002, 0.003, 0.004, 0.005,
|
|
89
|
+
0.006, 0.007, 0.008, 0.009, 0.010]]
|
|
90
|
+
stream.initialize(configurations, max_steps=10, seed=42)
|
|
91
|
+
suggestion = stream.suggest()
|
|
92
|
+
# Train the suggested configuration locally, then report validation loss:
|
|
93
|
+
# stream.report(suggestion['idx'], validation_loss)
|
|
94
|
+
# Continue with suggest/report or report_and_suggest as needed.
|
|
95
|
+
finally:
|
|
96
|
+
stream.close()
|
|
97
|
+
finally:
|
|
98
|
+
api.terminate_session(session_id)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
`initialize` and `reset` default to `effort="medium"`. A report records one
|
|
102
|
+
observation; `max_steps` limits observations per model, not optimizer updates.
|
|
103
|
+
Use `status()` to inspect progress and rankings. `reset(...)` starts a fresh
|
|
104
|
+
run after the previous run completes. `api.list_sessions()` lists your sessions;
|
|
105
|
+
`api.wait_for_termination(session_id)` waits for a release to complete.
|
|
106
|
+
|
|
107
|
+
## Datasets and training budgets
|
|
108
|
+
|
|
109
|
+
Each demo starts with ten sampled model configurations and allows up to 50
|
|
110
|
+
validation observations across them, with at most ten observations per model.
|
|
111
|
+
After tuning, all models finish to the same update cap for comparison.
|
|
112
|
+
Classification ranks use validation cross-entropy; regression ranks use MSE in
|
|
113
|
+
original target units. Lower is better. Test data is used only for final reporting.
|
|
114
|
+
|
|
115
|
+
| Dataset | Updates per observation | Maximum updates per model | Source |
|
|
116
|
+
| --- | ---: | ---: | --- |
|
|
117
|
+
| `iris` | 10 | 100 | scikit-learn |
|
|
118
|
+
| `wine` | 200 | 2,000 | scikit-learn |
|
|
119
|
+
| `breast-cancer` | 30 | 300 | scikit-learn |
|
|
120
|
+
| `digits` | 60 | 600 | scikit-learn |
|
|
121
|
+
| `glass` | 100 | 1,000 | OpenML dataset 41 |
|
|
122
|
+
| `vehicle` | 60 | 600 | OpenML dataset 54 |
|
|
123
|
+
| `diabetes` | 10 | 100 | scikit-learn |
|
|
124
|
+
| `friedman` | 200 | 2,000 | Generated regression data |
|
|
125
|
+
| `linear-regression` | 200 | 2,000 | Generated regression data |
|
|
126
|
+
| `toy-lm` | 30 | 300 | Generated language teaching task |
|
|
127
|
+
| `toy-vla` | 200 | 2,000 | Generated vision/language teaching task |
|
|
128
|
+
|
|
129
|
+
Glass and vehicle download on first use. Ordinary datasets use a 70/15/15
|
|
130
|
+
train/validation/test split; wine uses 55/30/15. Identical language contexts are
|
|
131
|
+
grouped into one split. Preprocessing is fitted on training data only.
|
|
132
|
+
|
|
133
|
+
`--block-steps N` sets updates per observation and a maximum of `10*N` updates.
|
|
134
|
+
The defaults are bounded workload profiles, not guarantees of convergence.
|
|
135
|
+
Version 2 uses loss-based classification ranks; these are not comparable with
|
|
136
|
+
version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
|
|
137
|
+
`--split-seed`, and `--training-seed` allow independent overrides.
|
|
138
|
+
|
|
139
|
+
## Repeated benchmarks
|
|
140
|
+
|
|
141
|
+
```sh
|
|
142
|
+
python -m tuning_demo.stats --api-url https://YOUR_API --email you@example.com \
|
|
143
|
+
--datasets iris wine --runs 2
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Without dataset/run options, this runs all eleven datasets ten times, starting
|
|
147
|
+
at seed 42. Each trial starts with fresh models and tuning state. Sessions are
|
|
148
|
+
reused per dataset; `--session-scope run` reuses one session across datasets.
|
|
149
|
+
Service benchmarks run sequentially.
|
|
150
|
+
|
|
151
|
+
Local reference schedulers need no service account:
|
|
152
|
+
|
|
153
|
+
```sh
|
|
154
|
+
python -m tuning_demo --scheduler asha --dataset digits
|
|
155
|
+
python -m tuning_demo.stats --scheduler flaml --datasets iris wine --runs 2 --workers 2
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Available schedulers are `service` (default), `asha`, `median`, `pbt`, `pb2`,
|
|
159
|
+
and `flaml`. `--workers` applies only to local reference schedulers; it cannot
|
|
160
|
+
parallelize service benchmarks. Reference statistics default to available CPU
|
|
161
|
+
cores, capped by trial count; use `--workers 1` to limit resource use.
|
|
162
|
+
See [benchmark details](tuning_demo/benchmarking/README.md) for algorithm settings.
|
|
163
|
+
|
|
164
|
+
Results include configurations, seeds, observation histories, timings, losses,
|
|
165
|
+
and rankings at the tuning cutoff and after final training. Statistics report
|
|
166
|
+
the mean final rank of the model selected at the cutoff and the best final rank
|
|
167
|
+
among models already capped at that cutoff. Ties receive ranks such as 1, 1, 3;
|
|
168
|
+
unavailable measurements and failed trials do not count as zero. A failed service
|
|
169
|
+
trial skips the remaining trials for that dataset. The command exits nonzero
|
|
170
|
+
on trial or cleanup failure and preserves partial results.
|
|
171
|
+
|
|
172
|
+
Timing includes client training and service round trips; it is not a pure
|
|
173
|
+
measurement of service compute. The service ranking uses latest reported losses,
|
|
174
|
+
not predicted fully trained losses. PBT/PB2 compare evolved checkpoint lineages,
|
|
175
|
+
not the original fixed configurations.
|
|
176
|
+
|
|
177
|
+
## Custom datasets
|
|
178
|
+
|
|
179
|
+
```sh
|
|
180
|
+
python -m pip install 'tuning-client[demo,custom]'
|
|
181
|
+
python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.csv \
|
|
182
|
+
--target label --task classification --drop-columns id
|
|
183
|
+
python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.parquet \
|
|
184
|
+
--target price --task regression
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Use one row per independent example, unique column names, a non-missing scalar
|
|
188
|
+
target, and numeric or categorical input columns. Drop identifiers and columns
|
|
189
|
+
that leak the target. Custom datasets default to 100 updates per observation.
|
|
190
|
+
Files stay local. Use an appropriate split outside this demo for time series,
|
|
191
|
+
grouped observations, or other data that requires special leakage controls.
|
|
192
|
+
|
|
193
|
+
## License and third-party notices
|
|
194
|
+
|
|
195
|
+
Original SDK and demo code is licensed under the [MIT License](LICENSE).
|
|
196
|
+
|
|
197
|
+
The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
|
|
198
|
+
and FLAML (MIT). Their license texts and attribution are included in the package;
|
|
199
|
+
see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
tuning_client/__init__.py,sha256=SNLJubYpRMTGCX7FEH6F6EOqiExwFrJukplG-fwuWPg,206
|
|
2
|
+
tuning_client/sdk.py,sha256=95Nm81Y4Es3bppVD3mBmjuiWffeWNI2pyDcswTe2Yzw,16476
|
|
3
|
+
tuning_client-0.1.0.dist-info/licenses/LICENSE,sha256=tcjmdWdr9BPAMro02ZgR3AYNfP8u6vrLQ2NX2yKjkTo,1082
|
|
4
|
+
tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-FLAML.txt,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
|
|
5
|
+
tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-Ray.txt,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
|
|
6
|
+
tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/THIRD_PARTY.md,sha256=Fs_-pUjT5uti5hX9-dx2FSk3kRxidGtLGcsOiFWrYkA,2771
|
|
7
|
+
tuning_demo/__init__.py,sha256=cze_I_quWJFfMhrDLwMASbUo2gczy360k2ZkmQlM7Cc,75
|
|
8
|
+
tuning_demo/__main__.py,sha256=D2ZSKdOSvqATo1-Y9Y3vlALhCJBxGTZ8ADeDZ6xd8z8,14220
|
|
9
|
+
tuning_demo/budgets.py,sha256=4KEBngT1FgfKWnJng83bKFtZYBzj6SiMJei3kNCQKJA,1314
|
|
10
|
+
tuning_demo/datasets.py,sha256=ZJDBFRF28WCTp67xcL_BywoDfWKtzOXkX9lZ7BfIDtQ,5430
|
|
11
|
+
tuning_demo/stats.py,sha256=uHy-oBVimxVGZxhkwqs79yjsKUQUB5xHqalN6o-OC7g,18869
|
|
12
|
+
tuning_demo/timing.py,sha256=ok8qd7sgmHGZbZhbd4dMahQuaPfKMeD8RCfWBf4pe7s,4380
|
|
13
|
+
tuning_demo/training.py,sha256=odsfR8Z3ez19tC3cYsXzTkqfutn1BM9ZUOUW5J79r2w,12030
|
|
14
|
+
tuning_demo/benchmarking/LICENSE-FLAML.txt,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
|
|
15
|
+
tuning_demo/benchmarking/LICENSE-Ray.txt,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
|
|
16
|
+
tuning_demo/benchmarking/README.md,sha256=JktLh-E4vceOyvXrOiT7W8Ag3zmDECUIPocpHgWYyFA,8314
|
|
17
|
+
tuning_demo/benchmarking/THIRD_PARTY.md,sha256=Fs_-pUjT5uti5hX9-dx2FSk3kRxidGtLGcsOiFWrYkA,2771
|
|
18
|
+
tuning_demo/benchmarking/__init__.py,sha256=6sKJNuE38y4jUmivDrPbNkdVYcimh1CPBF0xDSAu-9s,2030
|
|
19
|
+
tuning_demo/benchmarking/asha.py,sha256=3o80EMiQVkSNLJik2YOTxZb8iz48jWP7GkueULSv2W8,1319
|
|
20
|
+
tuning_demo/benchmarking/flaml.py,sha256=sMKkVBmzA4EosBBk6mQkirVcDiua989Y_oEAw143TMQ,2958
|
|
21
|
+
tuning_demo/benchmarking/median.py,sha256=XBuTRjuXU8UlskdFMShthcirduKOJ4BzhKiFCwntEaQ,1846
|
|
22
|
+
tuning_demo/benchmarking/parallel.py,sha256=ZFHxPaScF74tLiJEK4E3F2-mV2c19aqZgr5yfK7wYOM,4662
|
|
23
|
+
tuning_demo/benchmarking/pb2.py,sha256=WzlbCA5Wv_xb7g5r9ibJ-hSV0KPh-gTxhZkHHDEBTx8,4873
|
|
24
|
+
tuning_demo/benchmarking/pb2_utils.py,sha256=SgSHHpCq8-nZPg1Bp2tPYZWAP28ejMdlsH0V-Nrdx5k,5260
|
|
25
|
+
tuning_demo/benchmarking/pbt.py,sha256=CyNpVOUvoxABTAldrSUg6g-M8pC7iNQ6cvlqiCSaDOY,2855
|
|
26
|
+
tuning_demo/benchmarking/population.py,sha256=N9dP8V0-Tlsx14ntv8Fz2AqQ59zAjP8isxtQBoOpuiA,1397
|
|
27
|
+
tuning_demo/benchmarking/runner.py,sha256=fLLqFjMYlkkOCH56IVrIT9dfHurpztVYd6l4peilScY,10400
|
|
28
|
+
tuning_examples/__init__.py,sha256=bh68kODvkRypPGtzBudtI3XsIWQXlu2eV-EnF4bWzZc,79
|
|
29
|
+
tuning_examples/custom.py,sha256=27GKItaWKp2-rh1v5JmQjCG55lC_Tf8li0mU-PWHzq4,4299
|
|
30
|
+
tuning_examples/telemetry.py,sha256=LID5W-L5AH43cEweiUGFO3oJiCpIiEFTVE963cXxbI4,918
|
|
31
|
+
tuning_examples/toys.py,sha256=gta_O5cD0PtQAhJ0a-FX2Wy7fpBMV_z9d0__XI7V-jY,5700
|
|
32
|
+
tuning_client-0.1.0.dist-info/METADATA,sha256=MyO24e54QWkipE1fqXqwlGMHCig8ZdAxmwUmYcAyjpo,8965
|
|
33
|
+
tuning_client-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
34
|
+
tuning_client-0.1.0.dist-info/entry_points.txt,sha256=zV6zFaC0421qU_ISa6PidaEsI-nN4kYbhgyWODIFdpA,65
|
|
35
|
+
tuning_client-0.1.0.dist-info/top_level.txt,sha256=0DcramCIEPfx1RHLQpHYq9khS1bavDbKJ0ldeopyph8,42
|
|
36
|
+
tuning_client-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tuning Client contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) Microsoft Corporation.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE
|