tuning-client 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. tuning_client/__init__.py +4 -0
  2. tuning_client/sdk.py +344 -0
  3. tuning_client-0.1.0.dist-info/METADATA +199 -0
  4. tuning_client-0.1.0.dist-info/RECORD +36 -0
  5. tuning_client-0.1.0.dist-info/WHEEL +5 -0
  6. tuning_client-0.1.0.dist-info/entry_points.txt +2 -0
  7. tuning_client-0.1.0.dist-info/licenses/LICENSE +21 -0
  8. tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-FLAML.txt +21 -0
  9. tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-Ray.txt +177 -0
  10. tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/THIRD_PARTY.md +49 -0
  11. tuning_client-0.1.0.dist-info/top_level.txt +3 -0
  12. tuning_demo/__init__.py +1 -0
  13. tuning_demo/__main__.py +240 -0
  14. tuning_demo/benchmarking/LICENSE-FLAML.txt +21 -0
  15. tuning_demo/benchmarking/LICENSE-Ray.txt +177 -0
  16. tuning_demo/benchmarking/README.md +135 -0
  17. tuning_demo/benchmarking/THIRD_PARTY.md +49 -0
  18. tuning_demo/benchmarking/__init__.py +32 -0
  19. tuning_demo/benchmarking/asha.py +32 -0
  20. tuning_demo/benchmarking/flaml.py +70 -0
  21. tuning_demo/benchmarking/median.py +41 -0
  22. tuning_demo/benchmarking/parallel.py +107 -0
  23. tuning_demo/benchmarking/pb2.py +94 -0
  24. tuning_demo/benchmarking/pb2_utils.py +139 -0
  25. tuning_demo/benchmarking/pbt.py +60 -0
  26. tuning_demo/benchmarking/population.py +33 -0
  27. tuning_demo/benchmarking/runner.py +167 -0
  28. tuning_demo/budgets.py +38 -0
  29. tuning_demo/datasets.py +98 -0
  30. tuning_demo/stats.py +319 -0
  31. tuning_demo/timing.py +80 -0
  32. tuning_demo/training.py +239 -0
  33. tuning_examples/__init__.py +1 -0
  34. tuning_examples/custom.py +78 -0
  35. tuning_examples/telemetry.py +26 -0
  36. tuning_examples/toys.py +123 -0
@@ -0,0 +1,4 @@
1
+ """Synchronous SDK: training frameworks can call it without an async event loop."""
2
+ from .sdk import TuningClient, ProtocolError, TuningSession
3
+
4
+ __all__ = ["TuningClient", "ProtocolError", "TuningSession"]
tuning_client/sdk.py ADDED
@@ -0,0 +1,344 @@
1
+ import base64
2
+ import json
3
+ import ssl
4
+ import time
5
+ from urllib.parse import urlsplit
6
+
7
+ import httpx
8
+ from websockets.exceptions import ConnectionClosed, InvalidHandshake
9
+ from websockets.sync.client import connect
10
+
11
+
12
+ class ProtocolError(RuntimeError):
13
+ """The server rejected a request or cannot safely resume this session."""
14
+
15
+
16
+ def validate_transport(url, secure_scheme, allow_insecure):
17
+ parts = urlsplit(url)
18
+ if (not parts.hostname or parts.username or parts.password or parts.query or parts.fragment):
19
+ raise ValueError("Transport URL must have a host and no credentials, query or fragment")
20
+ if parts.scheme == secure_scheme:
21
+ return
22
+ if (allow_insecure and parts.scheme == secure_scheme[:-1]
23
+ and parts.hostname in ("localhost", "127.0.0.1", "::1")):
24
+ return
25
+ raise ValueError(f"{secure_scheme.upper()} is required; allow_insecure=True permits loopback debugging only")
26
+
27
+
28
+ class TuningClient:
29
+ def __init__(self, base_url, access_token=None, *, timeout=30, transport=None, allow_insecure=False):
30
+ validate_transport(base_url, "https", allow_insecure)
31
+ self.allow_insecure = allow_insecure
32
+ self.http = httpx.Client(base_url=base_url.rstrip("/"), timeout=timeout, transport=transport)
33
+ if access_token:
34
+ self.http.headers["Authorization"] = f"Bearer {access_token}"
35
+
36
+ def _request(self, method, path, **kwargs):
37
+ response = self.http.request(method, path, **kwargs)
38
+ try:
39
+ response.raise_for_status()
40
+ except httpx.HTTPStatusError as exc:
41
+ # FastAPI's human-readable reason is otherwise hidden by httpx.
42
+ # Do not include structured validation errors: those can echo input.
43
+ try:
44
+ payload = response.json()
45
+ except ValueError:
46
+ payload = None
47
+ detail = payload.get("detail") if isinstance(payload, dict) else None
48
+ if isinstance(detail, str) and detail:
49
+ raise httpx.HTTPStatusError(
50
+ f"{exc}\nAPI detail: {detail[:1000]}",
51
+ request=exc.request, response=exc.response,
52
+ ) from None
53
+ raise
54
+ return response.json()
55
+
56
+ def register(self, email, password, *, challenge_token=""):
57
+ """Public beta signup normally uses /signup in the browser."""
58
+ return self._request("POST", "/auth/register", json={"email": email, "password": password,
59
+ "challenge_token": challenge_token})
60
+
61
+ def verify_email(self, token):
62
+ return self._request("POST", "/auth/verify", json={"token": token})
63
+
64
+ def list_sessions(self):
65
+ return self._request("GET", "/sessions")
66
+
67
+ def wait_for_termination(self, session_id, *, timeout=180, poll_interval=2):
68
+ """Wait until cleanup releases the user's single-session allowance."""
69
+ deadline = time.monotonic() + timeout
70
+ while any(s["session_id"] == session_id for s in self.list_sessions()):
71
+ remaining = deadline - time.monotonic()
72
+ if remaining <= 0:
73
+ raise TimeoutError(f"Cleanup pending for {session_id}; retry DELETE before creating another session")
74
+ time.sleep(min(poll_interval, remaining))
75
+
76
+ def login(self, email, password):
77
+ result = self._request("POST", "/auth/login", json={"email": email, "password": password})
78
+ self.http.headers["Authorization"] = f"Bearer {result['access_token']}"
79
+
80
+ def create_session(self):
81
+ # Never automatically retry creation: the API has no idempotency key.
82
+ return self._request("POST", "/sessions")["session_id"]
83
+
84
+ def terminate_session(self, session_id):
85
+ """Delete the VM, unlike TuningSession.close(), which only disconnects."""
86
+ return self._request("DELETE", f"/sessions/{session_id}")
87
+
88
+ def connection_info(self, session_id):
89
+ return self._request("POST", f"/sessions/{session_id}/stream-token")
90
+
91
+ def stream_token(self, session_id):
92
+ return self.connection_info(session_id)["stream_token"]
93
+
94
+ def retry_session(self, session_id):
95
+ """Retry/reconcile provisioning of this session's existing VM identity."""
96
+ return self._request("POST", f"/sessions/{session_id}/retry")
97
+
98
+ def wait_for_endpoint(self, session_id, *, timeout=600, poll_interval=2, on_progress=None):
99
+ """Wait for an assigned secure endpoint; the API status is not a readiness signal."""
100
+ deadline = time.monotonic() + timeout
101
+ next_report = deadline - timeout
102
+ while True:
103
+ session = self.connection_info(session_id)
104
+ if session["status"] in ("STOPPING", "FINISHED"):
105
+ raise RuntimeError(f"Session {session_id} is {session['status']}")
106
+ if session.get("stream_url"):
107
+ validate_transport(session["stream_url"], "wss", self.allow_insecure)
108
+ return session
109
+ if session["status"] == "FAILED":
110
+ raise TimeoutError(f"Provisioning attempt failed for session {session_id}")
111
+ now = time.monotonic()
112
+ remaining = deadline - now
113
+ if on_progress and now >= next_report:
114
+ on_progress(f"Queued, position {session.get('queue_position', '?')}; cancel with terminate_session()"
115
+ if session.get('queued') else "Preparing your session")
116
+ next_report = now + 10
117
+ if remaining <= 0:
118
+ raise TimeoutError(f"Session {session_id} has no assigned endpoint yet")
119
+ time.sleep(min(poll_interval, remaining))
120
+
121
+ def wait_until_running(self, session_id, **kwargs):
122
+ """Compatibility alias; use tuning_session() to check actual readiness."""
123
+ return self.wait_for_endpoint(session_id, **kwargs)
124
+
125
+ def tuning_session(self, session_id, *, stream_url=None, startup_timeout=600,
126
+ provisioning_retries=2, poll_interval=2, on_progress=None, **kwargs):
127
+ """Connect directly to the VM, retrying provisioning of the SAME session.
128
+
129
+ startup_timeout bounds each attempt. No attempt creates another session.
130
+ An explicit endpoint override bypasses endpoint discovery.
131
+ """
132
+ if startup_timeout <= 0 or provisioning_retries < 0 or poll_interval <= 0:
133
+ raise ValueError("Timeout/poll interval must be positive and retries non-negative")
134
+ def progress(message):
135
+ if on_progress:
136
+ on_progress(message)
137
+
138
+ network_errors = (OSError, TimeoutError, ConnectionClosed, InvalidHandshake, httpx.TransportError)
139
+ for attempt in range(provisioning_retries + 1):
140
+ session = None
141
+ deadline = time.monotonic() + startup_timeout
142
+ try:
143
+ tls_options = {}
144
+ if stream_url:
145
+ url = stream_url
146
+ else:
147
+ info = self.wait_for_endpoint(session_id, timeout=startup_timeout, poll_interval=poll_interval,
148
+ on_progress=on_progress)
149
+ progress("Session assigned")
150
+ url = info["stream_url"]
151
+ if info.get("tls_certificate"):
152
+ context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
153
+ context.load_verify_locations(cadata=info["tls_certificate"])
154
+ tls_options = {"ssl_context": context, "tls_server_name": info["tls_server_name"]}
155
+ session = TuningSession(session_id, url, lambda: self.stream_token(session_id),
156
+ allow_insecure=self.allow_insecure, **tls_options, **kwargs)
157
+ connection_timeout = session.timeout
158
+ reported_error = None
159
+ progress("Connecting to session")
160
+ while True:
161
+ remaining = deadline - time.monotonic()
162
+ if remaining <= 0:
163
+ raise TimeoutError("VM stream did not become ready")
164
+ session.timeout = min(connection_timeout, remaining)
165
+ try:
166
+ session.connect()
167
+ session.timeout = connection_timeout
168
+ progress("Connected")
169
+ return session
170
+ except ssl.SSLCertVerificationError:
171
+ raise
172
+ except network_errors as exc:
173
+ error_type = type(exc).__name__
174
+ if reported_error != error_type:
175
+ progress("Waiting for connection")
176
+ reported_error = error_type
177
+ session.close()
178
+ time.sleep(min(poll_interval, max(0, deadline - time.monotonic())))
179
+ except ssl.SSLCertVerificationError:
180
+ raise
181
+ except network_errors:
182
+ if session:
183
+ session.close()
184
+ if attempt == provisioning_retries:
185
+ raise TimeoutError(
186
+ f"Session {session_id} did not become reachable after {attempt + 1} attempts; "
187
+ "retry this same session ID"
188
+ ) from None
189
+ progress("Retrying session setup")
190
+ self.retry_session(session_id)
191
+ except Exception:
192
+ if session:
193
+ session.close()
194
+ raise
195
+
196
+ def close(self):
197
+ self.http.close()
198
+
199
+ def __enter__(self):
200
+ return self
201
+
202
+ def __exit__(self, *_):
203
+ self.close()
204
+
205
+
206
+ class TuningSession:
207
+ """One sequential caller per session; reconnect state lives in this object.
208
+
209
+ token_provider is called on every new socket. A pending request is replayed
210
+ or resent with the SAME sequence; training must happen outside request().
211
+ Reuse this object after a network interruption. Process-restart recovery
212
+ would also require saving the client's model/optimizer/RNG state.
213
+ """
214
+
215
+ def __init__(self, session_id, stream_url, token_provider, *, timeout=30, retries=4, allow_insecure=False,
216
+ ssl_context=None, tls_server_name=None):
217
+ validate_transport(stream_url, "wss", allow_insecure)
218
+ if ssl_context is not None and (not ssl_context.check_hostname or ssl_context.verify_mode != ssl.CERT_REQUIRED):
219
+ raise ValueError("TLS certificate and hostname verification must remain enabled")
220
+ self.ssl_context = ssl_context
221
+ self.tls_server_name = tls_server_name
222
+ self.session_id = session_id
223
+ self.stream_url = stream_url
224
+ self.token_provider = token_provider
225
+ self.timeout = timeout
226
+ self.retries = retries
227
+ self.last_received_seq = 0
228
+ self.last_request_retried = False
229
+ self._socket = None
230
+ self._pending = None
231
+ self._replayed = None
232
+
233
+ def _receive(self):
234
+ message = json.loads(self._socket.recv(timeout=self.timeout))
235
+ if not isinstance(message, dict):
236
+ raise ProtocolError("Expected a JSON object")
237
+ if message.get("type") == "error":
238
+ if message.get("code") == "invalid_job":
239
+ self._pending = None # rejected jobs do not consume a sequence
240
+ raise ProtocolError(f"{message.get('code')}: {message.get('detail', '')}")
241
+ return message
242
+
243
+ def connect(self):
244
+ if self._socket is not None:
245
+ return self
246
+ try:
247
+ self._socket = connect(self.stream_url, open_timeout=self.timeout, close_timeout=2,
248
+ additional_headers={"Authorization": f"Bearer {self.token_provider()}"},
249
+ ssl=self.ssl_context, server_hostname=self.tls_server_name)
250
+ self._socket.send(json.dumps({"type": "resume", "session_id": self.session_id,
251
+ "last_received_seq": self.last_received_seq}))
252
+ response = self._receive()
253
+ if response.get("type") != "resume_ok" or response.get("session_id") != self.session_id:
254
+ raise ProtocolError("Invalid resume response")
255
+ server_seq = response.get("server_seq")
256
+ maximum = self.last_received_seq + (1 if self._pending else 0)
257
+ if type(server_seq) is not int or not self.last_received_seq <= server_seq <= maximum:
258
+ raise ProtocolError("Session state lost or another client advanced this session")
259
+ if server_seq > self.last_received_seq:
260
+ self._replayed = self._receive()
261
+ return self
262
+ except Exception:
263
+ self.close()
264
+ raise
265
+
266
+ def request(self, operation, **payload):
267
+ job = {"job": "tuning", "op": operation, **payload}
268
+ encoded = base64.b64encode(json.dumps(job, allow_nan=False).encode()).decode()
269
+ frame = {"type": "data", "seq": self.last_received_seq + 1, "data": encoded}
270
+ if self._pending is not None and self._pending != frame:
271
+ raise ProtocolError("Retry the pending request before submitting another operation")
272
+ self.last_request_retried = self._pending is not None
273
+ self._pending = frame
274
+ for attempt in range(self.retries + 1):
275
+ try:
276
+ self.connect()
277
+ if self._replayed is not None:
278
+ response, self._replayed = self._replayed, None
279
+ else:
280
+ self._socket.send(json.dumps(frame))
281
+ response = self._receive()
282
+ if response.get("type") != "result" or response.get("seq") != frame["seq"]:
283
+ raise ProtocolError("Unexpected result sequence")
284
+ result = json.loads(base64.b64decode(response["data"], validate=True))
285
+ if not isinstance(result, dict) or result.get("job") != "tuning" or result.get("op") != operation:
286
+ raise ProtocolError("Unexpected tuning result")
287
+ self.last_received_seq = frame["seq"]
288
+ self.last_request_retried |= bool(response.get("replayed", False))
289
+ self._pending = None
290
+ return result
291
+ except ssl.SSLCertVerificationError:
292
+ self.close()
293
+ raise
294
+ except (OSError, TimeoutError, ConnectionClosed, InvalidHandshake, httpx.TransportError):
295
+ self.last_request_retried = True
296
+ self.close()
297
+ if attempt == self.retries:
298
+ raise ConnectionError("Stream retries exhausted; retry the same operation on this object") from None
299
+ time.sleep(min(0.5 * 2**attempt, 5))
300
+ except Exception:
301
+ self.close()
302
+ raise
303
+
304
+ def initialize(self, configs, *, max_steps, seed=None, effort="medium"):
305
+ """Budgets count observations, not client optimizer updates."""
306
+ return self.request("initialize", configs=configs,
307
+ max_steps=max_steps, seed=seed, effort=effort)
308
+
309
+ def suggest(self):
310
+ return self.request("suggest")
311
+
312
+ def reset(self, configs, *, max_steps, seed=None, effort="medium"):
313
+ """Start fresh native state after completing the previous run's budget."""
314
+ return self.request("reset", configs=configs,
315
+ max_steps=max_steps, seed=seed, effort=effort)
316
+
317
+ def report(self, idx, error, steps=None):
318
+ """Report one error; optional steps is the model's cumulative pull count."""
319
+ fields = {} if steps is None else {"steps": steps}
320
+ return self.request("report", idx=idx, error=error, **fields)
321
+
322
+ def finish(self, final_errors=None):
323
+ fields = {} if final_errors is None else {"final_errors": final_errors}
324
+ return self.request("finish", **fields)
325
+
326
+ def report_and_suggest(self, idx, error, steps=None):
327
+ """Accept one observation and return the next suggestion in one reply."""
328
+ fields = {} if steps is None else {"steps": steps}
329
+ return self.request("report_and_suggest", idx=idx, error=error, **fields)
330
+
331
+ def status(self):
332
+ return self.request("status")
333
+
334
+ def close(self):
335
+ if self._socket is not None:
336
+ self._socket.close()
337
+ self._socket = None
338
+ self._replayed = None
339
+
340
+ def __enter__(self):
341
+ return self.connect()
342
+
343
+ def __exit__(self, *_):
344
+ self.close()
@@ -0,0 +1,199 @@
1
+ Metadata-Version: 2.4
2
+ Name: tuning-client
3
+ Version: 0.1.0
4
+ Summary: Python SDK and neural-network tuning demo for the hyperopt.dev AutoML service
5
+ License-Expression: MIT AND Apache-2.0
6
+ Project-URL: Homepage, https://hyperopt.dev
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ License-File: tuning_demo/benchmarking/LICENSE-FLAML.txt
11
+ License-File: tuning_demo/benchmarking/LICENSE-Ray.txt
12
+ License-File: tuning_demo/benchmarking/THIRD_PARTY.md
13
+ Requires-Dist: httpx<1,>=0.27
14
+ Requires-Dist: websockets<18,>=15
15
+ Provides-Extra: demo
16
+ Requires-Dist: numpy<3,>=1.26; extra == "demo"
17
+ Requires-Dist: scikit-learn<2,>=1.5; extra == "demo"
18
+ Requires-Dist: torch<3,>=2.5; extra == "demo"
19
+ Provides-Extra: custom
20
+ Requires-Dist: pandas<3,>=2.2; extra == "custom"
21
+ Requires-Dist: pyarrow<24,>=18; extra == "custom"
22
+ Dynamic: license-file
23
+
24
+ # Tuning Client Python SDK and demo
25
+
26
+ Tuning Client is the Python SDK and demo for the
27
+ [hyperopt.dev AutoML service](https://hyperopt.dev).
28
+
29
+ Use the Python SDK to request tuning suggestions while keeping model training,
30
+ datasets, and model weights on your computer. The optional demo trains small
31
+ PyTorch models and compares tuning strategies on classification, regression,
32
+ and teaching datasets.
33
+
34
+ **Code provenance:** The SDK and demo code were generated with ChatGPT and
35
+ reviewed. Review does not guarantee correctness; evaluate the code and benchmark
36
+ results for your use case. The demos are examples, not production ML benchmarks.
37
+
38
+ ## Installation
39
+
40
+ Requires Python 3.11 or newer. Install the SDK alone or include the demo:
41
+
42
+ ```sh
43
+ python -m pip install tuning-client
44
+ python -m pip install 'tuning-client[demo]'
45
+ ```
46
+
47
+ ## Run the demo
48
+
49
+ Create an account using the service's signup page and verify your email. Replace
50
+ `https://YOUR_API` below with the service URL provided to you:
51
+
52
+ ```sh
53
+ python -m tuning_demo --api-url https://YOUR_API --email you@example.com --dataset digits
54
+ # The installed command is equivalent:
55
+ tuning-client-demo --api-url https://YOUR_API --email you@example.com --dataset iris
56
+ python -m tuning_demo --list-datasets
57
+ ```
58
+
59
+ The demo prompts for your password. You can instead set `COMPUTE_ACCESS_TOKEN`,
60
+ or `COMPUTE_EMAIL` and `COMPUTE_PASSWORD`. Set `COMPUTE_API_URL` to avoid passing
61
+ `--api-url` each time. Connections use HTTPS and verified TLS.
62
+
63
+ Service effort defaults to **medium**. Use `--effort easy` for lower effort;
64
+ the observation budget stays the same. `--interactive` offers another dataset
65
+ after each run. Sessions created by the demo are released automatically on
66
+ completion or failure. `--session-id ID` uses an existing unused session that
67
+ you remain responsible for releasing; `--keep-session` retains a new session.
68
+
69
+ Results are saved in `./results/`. Downloaded datasets are cached in
70
+ `~/.cache/tuning-client/`. Override these with `--output-dir` and `--data-dir`.
71
+ Optional usage events include a run ID, session ID, workload category, and
72
+ progress event; use `--no-telemetry` to disable them. Training data, file paths,
73
+ and model weights are not included in those events.
74
+
75
+ ## SDK usage
76
+
77
+ ```python
78
+ import os
79
+ from tuning_client import TuningClient
80
+
81
+ with TuningClient(os.environ['COMPUTE_API_URL'],
82
+ access_token=os.environ['COMPUTE_ACCESS_TOKEN']) as api:
83
+ session_id = api.create_session()
84
+ try:
85
+ stream = api.tuning_session(session_id)
86
+ try:
87
+ configurations = [{'learning_rate': rate} for rate in
88
+ [0.001, 0.002, 0.003, 0.004, 0.005,
89
+ 0.006, 0.007, 0.008, 0.009, 0.010]]
90
+ stream.initialize(configurations, max_steps=10, seed=42)
91
+ suggestion = stream.suggest()
92
+ # Train the suggested configuration locally, then report validation loss:
93
+ # stream.report(suggestion['idx'], validation_loss)
94
+ # Continue with suggest/report or report_and_suggest as needed.
95
+ finally:
96
+ stream.close()
97
+ finally:
98
+ api.terminate_session(session_id)
99
+ ```
100
+
101
+ `initialize` and `reset` default to `effort="medium"`. A report records one
102
+ observation; `max_steps` limits observations per model, not optimizer updates.
103
+ Use `status()` to inspect progress and rankings. `reset(...)` starts a fresh
104
+ run after the previous run completes. `api.list_sessions()` lists your sessions;
105
+ `api.wait_for_termination(session_id)` waits for a release to complete.
106
+
107
+ ## Datasets and training budgets
108
+
109
+ Each demo starts with ten sampled model configurations and allows up to 50
110
+ validation observations across them, with at most ten observations per model.
111
+ After tuning, all models finish to the same update cap for comparison.
112
+ Classification ranks use validation cross-entropy; regression ranks use MSE in
113
+ original target units. Lower is better. Test data is used only for final reporting.
114
+
115
+ | Dataset | Updates per observation | Maximum updates per model | Source |
116
+ | --- | ---: | ---: | --- |
117
+ | `iris` | 10 | 100 | scikit-learn |
118
+ | `wine` | 200 | 2,000 | scikit-learn |
119
+ | `breast-cancer` | 30 | 300 | scikit-learn |
120
+ | `digits` | 60 | 600 | scikit-learn |
121
+ | `glass` | 100 | 1,000 | OpenML dataset 41 |
122
+ | `vehicle` | 60 | 600 | OpenML dataset 54 |
123
+ | `diabetes` | 10 | 100 | scikit-learn |
124
+ | `friedman` | 200 | 2,000 | Generated regression data |
125
+ | `linear-regression` | 200 | 2,000 | Generated regression data |
126
+ | `toy-lm` | 30 | 300 | Generated language teaching task |
127
+ | `toy-vla` | 200 | 2,000 | Generated vision/language teaching task |
128
+
129
+ Glass and vehicle download on first use. Ordinary datasets use a 70/15/15
130
+ train/validation/test split; wine uses 55/30/15. Identical language contexts are
131
+ grouped into one split. Preprocessing is fitted on training data only.
132
+
133
+ `--block-steps N` sets updates per observation and a maximum of `10*N` updates.
134
+ The defaults are bounded workload profiles, not guarantees of convergence.
135
+ Version 2 uses loss-based classification ranks; these are not comparable with
136
+ version 1 accuracy-error ranks. `--seed` sets all seeds; `--config-seed`,
137
+ `--split-seed`, and `--training-seed` allow independent overrides.
138
+
139
+ ## Repeated benchmarks
140
+
141
+ ```sh
142
+ python -m tuning_demo.stats --api-url https://YOUR_API --email you@example.com \
143
+ --datasets iris wine --runs 2
144
+ ```
145
+
146
+ Without dataset/run options, this runs all eleven datasets ten times, starting
147
+ at seed 42. Each trial starts with fresh models and tuning state. Sessions are
148
+ reused per dataset; `--session-scope run` reuses one session across datasets.
149
+ Service benchmarks run sequentially.
150
+
151
+ Local reference schedulers need no service account:
152
+
153
+ ```sh
154
+ python -m tuning_demo --scheduler asha --dataset digits
155
+ python -m tuning_demo.stats --scheduler flaml --datasets iris wine --runs 2 --workers 2
156
+ ```
157
+
158
+ Available schedulers are `service` (default), `asha`, `median`, `pbt`, `pb2`,
159
+ and `flaml`. `--workers` applies only to local reference schedulers; it cannot
160
+ parallelize service benchmarks. Reference statistics default to available CPU
161
+ cores, capped by trial count; use `--workers 1` to limit resource use.
162
+ See [benchmark details](tuning_demo/benchmarking/README.md) for algorithm settings.
163
+
164
+ Results include configurations, seeds, observation histories, timings, losses,
165
+ and rankings at the tuning cutoff and after final training. Statistics report
166
+ the mean final rank of the model selected at the cutoff and the best final rank
167
+ among models already capped at that cutoff. Ties receive ranks such as 1, 1, 3;
168
+ unavailable measurements and failed trials do not count as zero. A failed service
169
+ trial skips the remaining trials for that dataset. The command exits nonzero
170
+ on trial or cleanup failure and preserves partial results.
171
+
172
+ Timing includes client training and service round trips; it is not a pure
173
+ measurement of service compute. The service ranking uses latest reported losses,
174
+ not predicted fully trained losses. PBT/PB2 compare evolved checkpoint lineages,
175
+ not the original fixed configurations.
176
+
177
+ ## Custom datasets
178
+
179
+ ```sh
180
+ python -m pip install 'tuning-client[demo,custom]'
181
+ python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.csv \
182
+ --target label --task classification --drop-columns id
183
+ python -m tuning_demo --api-url https://YOUR_API --custom-data ./data.parquet \
184
+ --target price --task regression
185
+ ```
186
+
187
+ Use one row per independent example, unique column names, a non-missing scalar
188
+ target, and numeric or categorical input columns. Drop identifiers and columns
189
+ that leak the target. Custom datasets default to 100 updates per observation.
190
+ Files stay local. Use an appropriate split outside this demo for time series,
191
+ grouped observations, or other data that requires special leakage controls.
192
+
193
+ ## License and third-party notices
194
+
195
+ Original SDK and demo code is licensed under the [MIT License](LICENSE).
196
+
197
+ The benchmark implementations include adaptations of Ray Tune (Apache-2.0)
198
+ and FLAML (MIT). Their license texts and attribution are included in the package;
199
+ see [third-party notices](tuning_demo/benchmarking/THIRD_PARTY.md).
@@ -0,0 +1,36 @@
1
+ tuning_client/__init__.py,sha256=SNLJubYpRMTGCX7FEH6F6EOqiExwFrJukplG-fwuWPg,206
2
+ tuning_client/sdk.py,sha256=95Nm81Y4Es3bppVD3mBmjuiWffeWNI2pyDcswTe2Yzw,16476
3
+ tuning_client-0.1.0.dist-info/licenses/LICENSE,sha256=tcjmdWdr9BPAMro02ZgR3AYNfP8u6vrLQ2NX2yKjkTo,1082
4
+ tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-FLAML.txt,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
5
+ tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/LICENSE-Ray.txt,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
6
+ tuning_client-0.1.0.dist-info/licenses/tuning_demo/benchmarking/THIRD_PARTY.md,sha256=Fs_-pUjT5uti5hX9-dx2FSk3kRxidGtLGcsOiFWrYkA,2771
7
+ tuning_demo/__init__.py,sha256=cze_I_quWJFfMhrDLwMASbUo2gczy360k2ZkmQlM7Cc,75
8
+ tuning_demo/__main__.py,sha256=D2ZSKdOSvqATo1-Y9Y3vlALhCJBxGTZ8ADeDZ6xd8z8,14220
9
+ tuning_demo/budgets.py,sha256=4KEBngT1FgfKWnJng83bKFtZYBzj6SiMJei3kNCQKJA,1314
10
+ tuning_demo/datasets.py,sha256=ZJDBFRF28WCTp67xcL_BywoDfWKtzOXkX9lZ7BfIDtQ,5430
11
+ tuning_demo/stats.py,sha256=uHy-oBVimxVGZxhkwqs79yjsKUQUB5xHqalN6o-OC7g,18869
12
+ tuning_demo/timing.py,sha256=ok8qd7sgmHGZbZhbd4dMahQuaPfKMeD8RCfWBf4pe7s,4380
13
+ tuning_demo/training.py,sha256=odsfR8Z3ez19tC3cYsXzTkqfutn1BM9ZUOUW5J79r2w,12030
14
+ tuning_demo/benchmarking/LICENSE-FLAML.txt,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
15
+ tuning_demo/benchmarking/LICENSE-Ray.txt,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
16
+ tuning_demo/benchmarking/README.md,sha256=JktLh-E4vceOyvXrOiT7W8Ag3zmDECUIPocpHgWYyFA,8314
17
+ tuning_demo/benchmarking/THIRD_PARTY.md,sha256=Fs_-pUjT5uti5hX9-dx2FSk3kRxidGtLGcsOiFWrYkA,2771
18
+ tuning_demo/benchmarking/__init__.py,sha256=6sKJNuE38y4jUmivDrPbNkdVYcimh1CPBF0xDSAu-9s,2030
19
+ tuning_demo/benchmarking/asha.py,sha256=3o80EMiQVkSNLJik2YOTxZb8iz48jWP7GkueULSv2W8,1319
20
+ tuning_demo/benchmarking/flaml.py,sha256=sMKkVBmzA4EosBBk6mQkirVcDiua989Y_oEAw143TMQ,2958
21
+ tuning_demo/benchmarking/median.py,sha256=XBuTRjuXU8UlskdFMShthcirduKOJ4BzhKiFCwntEaQ,1846
22
+ tuning_demo/benchmarking/parallel.py,sha256=ZFHxPaScF74tLiJEK4E3F2-mV2c19aqZgr5yfK7wYOM,4662
23
+ tuning_demo/benchmarking/pb2.py,sha256=WzlbCA5Wv_xb7g5r9ibJ-hSV0KPh-gTxhZkHHDEBTx8,4873
24
+ tuning_demo/benchmarking/pb2_utils.py,sha256=SgSHHpCq8-nZPg1Bp2tPYZWAP28ejMdlsH0V-Nrdx5k,5260
25
+ tuning_demo/benchmarking/pbt.py,sha256=CyNpVOUvoxABTAldrSUg6g-M8pC7iNQ6cvlqiCSaDOY,2855
26
+ tuning_demo/benchmarking/population.py,sha256=N9dP8V0-Tlsx14ntv8Fz2AqQ59zAjP8isxtQBoOpuiA,1397
27
+ tuning_demo/benchmarking/runner.py,sha256=fLLqFjMYlkkOCH56IVrIT9dfHurpztVYd6l4peilScY,10400
28
+ tuning_examples/__init__.py,sha256=bh68kODvkRypPGtzBudtI3XsIWQXlu2eV-EnF4bWzZc,79
29
+ tuning_examples/custom.py,sha256=27GKItaWKp2-rh1v5JmQjCG55lC_Tf8li0mU-PWHzq4,4299
30
+ tuning_examples/telemetry.py,sha256=LID5W-L5AH43cEweiUGFO3oJiCpIiEFTVE963cXxbI4,918
31
+ tuning_examples/toys.py,sha256=gta_O5cD0PtQAhJ0a-FX2Wy7fpBMV_z9d0__XI7V-jY,5700
32
+ tuning_client-0.1.0.dist-info/METADATA,sha256=MyO24e54QWkipE1fqXqwlGMHCig8ZdAxmwUmYcAyjpo,8965
33
+ tuning_client-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
34
+ tuning_client-0.1.0.dist-info/entry_points.txt,sha256=zV6zFaC0421qU_ISa6PidaEsI-nN4kYbhgyWODIFdpA,65
35
+ tuning_client-0.1.0.dist-info/top_level.txt,sha256=0DcramCIEPfx1RHLQpHYq9khS1bavDbKJ0ldeopyph8,42
36
+ tuning_client-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ tuning-client-demo = tuning_demo.__main__:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Tuning Client contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) Microsoft Corporation.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE