euler-train 2.7.0__tar.gz → 2.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {euler_train-2.7.0 → euler_train-2.9.0}/PKG-INFO +1 -1
- {euler_train-2.7.0 → euler_train-2.9.0}/README.md +42 -7
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/__init__.py +2 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/stream.py +206 -57
- euler_train-2.9.0/euler_train/stream_check_cli.py +111 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train.egg-info/PKG-INFO +1 -1
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train.egg-info/SOURCES.txt +2 -0
- euler_train-2.9.0/euler_train.egg-info/entry_points.txt +2 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/pyproject.toml +4 -1
- {euler_train-2.7.0 → euler_train-2.9.0}/tests/test_runlog.py +187 -4
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/_dataset_contract.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/architecture.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/environment.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/git_info.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/outputs.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/run.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/serialization.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train/slurm.py +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train.egg-info/dependency_links.txt +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train.egg-info/requires.txt +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/euler_train.egg-info/top_level.txt +0 -0
- {euler_train-2.7.0 → euler_train-2.9.0}/setup.cfg +0 -0
|
@@ -64,9 +64,9 @@ run = euler_train.init(
|
|
|
64
64
|
config=cfg,
|
|
65
65
|
stream={
|
|
66
66
|
"base_url": os.environ["EULER_VIEW_BASE_URL"],
|
|
67
|
-
"model_id": 42,
|
|
68
67
|
"api_token": os.environ["EULER_VIEW_API_TOKEN"],
|
|
69
68
|
"stream_attach_token": os.environ.get("EULER_VIEW_STREAM_ATTACH_TOKEN"),
|
|
69
|
+
"model_id": 42, # only needed for the SLURM fallback path
|
|
70
70
|
},
|
|
71
71
|
)
|
|
72
72
|
```
|
|
@@ -74,8 +74,8 @@ run = euler_train.init(
|
|
|
74
74
|
What each field is for:
|
|
75
75
|
|
|
76
76
|
- `base_url`: Euler View server base URL, for example `https://view.example.com`.
|
|
77
|
-
- `model_id`: Euler View model that should receive this run.
|
|
78
|
-
- `api_token` or `access_token`: credential used to open a stream session. This is
|
|
77
|
+
- `model_id`: Euler View model that should receive this run. This is optional when `stream_attach_token` is provided, and required for the SLURM fallback path.
|
|
78
|
+
- `api_token` or `access_token`: credential used to open a stream session. This is required whenever the package needs to negotiate its own short-lived ingest token, even if `stream_attach_token` is also provided.
|
|
79
79
|
- `stream_attach_token`: optional opaque launch-attachment token. When present, the producer sends it during the session handshake and Euler View attaches the stream to that launch directly. This is the preferred path for interactive or otherwise non-SLURM launches. Internally Euler View currently maps this token to `model_launches.id`, but the package treats it as an opaque token.
|
|
80
80
|
- `stream_token`: optional pre-issued ingest token. If you pass this, the package skips the session handshake entirely and posts events directly to the ingest endpoint.
|
|
81
81
|
- `datasource_id`: optional Euler View datasource hint for the run record.
|
|
@@ -85,17 +85,52 @@ What each field is for:
|
|
|
85
85
|
How the handshake works:
|
|
86
86
|
|
|
87
87
|
1. `euler_train` creates the local run directory and writes `meta.json`, `config.json`, and the other standard files as usual.
|
|
88
|
-
2. When the first stream flush happens, the package requests a session from Euler View at
|
|
88
|
+
2. When the first stream flush happens, the package requests a session from Euler View at `POST /api/model-run-stream/session`.
|
|
89
89
|
3. If `stream_attach_token` is configured, it is sent as `streamAttachToken` and used as the explicit launch attachment key.
|
|
90
|
-
4. If no `stream_attach_token` is configured, the package falls back to the SLURM metadata already captured in `meta.json["slurm"]["job_id"]`.
|
|
90
|
+
4. If no `stream_attach_token` is configured, the package falls back to the SLURM metadata already captured in `meta.json["slurm"]["job_id"]`. In that fallback mode, `model_id` must be configured so Euler View knows which model namespace to match against.
|
|
91
91
|
5. Euler View returns a short-lived ingest token plus the canonical `streamAttachToken`; the package caches that returned token and reuses it on later session refreshes.
|
|
92
92
|
6. Subsequent events are sent to `/api/model-run-stream/ingest`.
|
|
93
93
|
|
|
94
|
+
`stream_attach_token` is only the attachment/correlation key. It does not authenticate the session request by itself. In the current implementation you still need `api_token` or `access_token` for the session handshake, unless you already have a pre-issued `stream_token`.
|
|
95
|
+
|
|
96
|
+
### Dry-run connectivity check
|
|
97
|
+
|
|
98
|
+
If you need to verify from a cluster node whether Euler View is reachable and whether the stream handshake would be accepted before starting a real run, the package exposes the same dry-run check used by the stream protocol:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
euler-train.stream.check \
|
|
102
|
+
--api-url https://view.example.com \
|
|
103
|
+
--api-key "$EULER_VIEW_API_TOKEN" \
|
|
104
|
+
--stream-attach-token "$EULER_VIEW_STREAM_ATTACH_TOKEN"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
This command calls `POST /api/model-run-stream/check`.
|
|
108
|
+
|
|
109
|
+
It validates:
|
|
110
|
+
|
|
111
|
+
- network reachability to Euler View
|
|
112
|
+
- API token validity and scope
|
|
113
|
+
- stream attachment resolution via `stream_attach_token`, when provided
|
|
114
|
+
- model existence, when the request uses the `model_id` + `--slurm-job-id` fallback path
|
|
115
|
+
- SLURM fallback matching via `--slurm-job-id`, when no attach token is provided
|
|
116
|
+
|
|
117
|
+
Useful flags:
|
|
118
|
+
|
|
119
|
+
- `--run-id`: override the dry-run run ID. If omitted, the CLI generates a temporary `stream-check-*` ID.
|
|
120
|
+
- `--stream-attach-token`: test the explicit launch attachment path.
|
|
121
|
+
- `--model-id`: required for the fallback SLURM matching path, optional when `--stream-attach-token` is used.
|
|
122
|
+
- `--slurm-job-id`: test the fallback SLURM matching path.
|
|
123
|
+
- `--datasource-id`: include the datasource hint that a real stream session would send.
|
|
124
|
+
- `--euler-train-dir` and `--run-dir`: include path hints in the dry-run payload.
|
|
125
|
+
- `--json`: print the raw JSON response for automation.
|
|
126
|
+
|
|
127
|
+
The dry-run check does not start a run and does not mint an ingest token. It only verifies that the real session handshake would be accepted with the same request shape.
|
|
128
|
+
|
|
94
129
|
What has to be configured where:
|
|
95
130
|
|
|
96
131
|
- In the training script or job wrapper: pass `stream={...}` to `euler_train.init(...)`.
|
|
97
|
-
- In Euler View / the launcher: if the run should attach to a specific launch regardless of SLURM availability, provide a `stream_attach_token` to the runtime and forward it into the `stream` config.
|
|
98
|
-
- Under plain SLURM launches: no extra attachment config is required as long as Euler View can uniquely match the run by `slurm.job_id
|
|
132
|
+
- In Euler View / the launcher: if the run should attach to a specific launch regardless of SLURM availability, provide a `stream_attach_token` to the runtime and forward it into the `stream` config. In that mode the producer does not need to know `model_id`.
|
|
133
|
+
- Under plain SLURM launches: no extra attachment config is required as long as Euler View can uniquely match the run by `slurm.job_id`, but `model_id` still has to be configured for the fallback lookup.
|
|
99
134
|
- If neither `stream_attach_token` nor a unique SLURM match is available, run streaming still works, but launch-to-run linking in Euler View may remain unresolved.
|
|
100
135
|
|
|
101
136
|
For new integrations, prefer `stream_attach_token`. The older `launch_id` / `launchId` config keys are still accepted as compatibility aliases, but new code should treat the attachment value as an opaque stream token rather than a launch ID.
|
|
@@ -12,6 +12,7 @@ from .stream import (
|
|
|
12
12
|
OutputStream,
|
|
13
13
|
OutputStreamConsumer,
|
|
14
14
|
StreamContext,
|
|
15
|
+
check_stream_handshake,
|
|
15
16
|
)
|
|
16
17
|
|
|
17
18
|
__all__ = [
|
|
@@ -20,6 +21,7 @@ __all__ = [
|
|
|
20
21
|
"OutputStream",
|
|
21
22
|
"OutputStreamConsumer",
|
|
22
23
|
"StreamContext",
|
|
24
|
+
"check_stream_handshake",
|
|
23
25
|
"init",
|
|
24
26
|
"Run",
|
|
25
27
|
"export_architecture",
|
|
@@ -11,7 +11,6 @@ from datetime import datetime
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any, Protocol
|
|
13
13
|
from urllib.error import HTTPError, URLError
|
|
14
|
-
from urllib.parse import quote
|
|
15
14
|
from urllib.request import Request, urlopen
|
|
16
15
|
|
|
17
16
|
from .serialization import _json_default
|
|
@@ -146,9 +145,9 @@ class EulerViewStreamConfig:
|
|
|
146
145
|
)
|
|
147
146
|
|
|
148
147
|
if self.stream_token is None:
|
|
149
|
-
if self.model_id is None:
|
|
148
|
+
if self.model_id is None and self.stream_attach_token is None:
|
|
150
149
|
raise ValueError(
|
|
151
|
-
"stream.model_id is required when stream.stream_token is
|
|
150
|
+
"stream.model_id is required when neither stream.stream_token nor stream.stream_attach_token is provided",
|
|
152
151
|
)
|
|
153
152
|
if self.api_token is None and self.access_token is None:
|
|
154
153
|
raise ValueError(
|
|
@@ -312,26 +311,20 @@ class EulerViewStreamConsumer:
|
|
|
312
311
|
if not force and now < self._next_retry_at:
|
|
313
312
|
return False
|
|
314
313
|
|
|
315
|
-
session_url =
|
|
316
|
-
|
|
317
|
-
|
|
314
|
+
session_url = f"{self.config.base_url}/api/model-run-stream/session"
|
|
315
|
+
session_body = _build_stream_session_body(
|
|
316
|
+
run_id=self._context.run_id,
|
|
317
|
+
model_id=self.config.model_id,
|
|
318
|
+
run_dir=self.config.run_dir or str(self._context.run_dir),
|
|
319
|
+
euler_train_dir=self.config.euler_train_dir or str(self._context.runs_dir),
|
|
320
|
+
stream_attach_token=(
|
|
321
|
+
self.config.stream_attach_token
|
|
322
|
+
or self._resolved_stream_attach_token
|
|
323
|
+
),
|
|
324
|
+
slurm_job_id=_extract_slurm_job_id(self._context.meta),
|
|
325
|
+
datasource_id=self.config.datasource_id,
|
|
326
|
+
session_expires_in_sec=self.config.session_expires_in_sec,
|
|
318
327
|
)
|
|
319
|
-
session_body: dict[str, Any] = {
|
|
320
|
-
"runDir": self.config.run_dir or str(self._context.run_dir),
|
|
321
|
-
"eulerTrainDir": self.config.euler_train_dir or str(self._context.runs_dir),
|
|
322
|
-
}
|
|
323
|
-
stream_attach_token = (
|
|
324
|
-
self.config.stream_attach_token or self._resolved_stream_attach_token
|
|
325
|
-
)
|
|
326
|
-
if stream_attach_token is not None:
|
|
327
|
-
session_body["streamAttachToken"] = stream_attach_token
|
|
328
|
-
slurm_job_id = _extract_slurm_job_id(self._context.meta)
|
|
329
|
-
if slurm_job_id is not None:
|
|
330
|
-
session_body["slurmJobId"] = slurm_job_id
|
|
331
|
-
if self.config.datasource_id is not None:
|
|
332
|
-
session_body["datasourceId"] = self.config.datasource_id
|
|
333
|
-
if self.config.session_expires_in_sec is not None:
|
|
334
|
-
session_body["expiresInSec"] = self.config.session_expires_in_sec
|
|
335
328
|
|
|
336
329
|
try:
|
|
337
330
|
payload = self._post_json(
|
|
@@ -372,44 +365,13 @@ class EulerViewStreamConsumer:
|
|
|
372
365
|
*,
|
|
373
366
|
token: str | None,
|
|
374
367
|
) -> dict[str, Any]:
|
|
375
|
-
|
|
376
|
-
"Accept": "application/json",
|
|
377
|
-
"Content-Type": "application/json",
|
|
378
|
-
}
|
|
379
|
-
if token:
|
|
380
|
-
headers["Authorization"] = f"Bearer {token}"
|
|
381
|
-
|
|
382
|
-
request = Request(
|
|
368
|
+
return _post_json_request(
|
|
383
369
|
url,
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
370
|
+
payload,
|
|
371
|
+
token=token,
|
|
372
|
+
timeout_sec=self.config.timeout_sec,
|
|
387
373
|
)
|
|
388
374
|
|
|
389
|
-
try:
|
|
390
|
-
with urlopen(request, timeout=self.config.timeout_sec) as response:
|
|
391
|
-
raw = response.read()
|
|
392
|
-
except HTTPError as exc:
|
|
393
|
-
message = _extract_http_error_message(exc)
|
|
394
|
-
raise RuntimeError(message) from exc
|
|
395
|
-
except URLError as exc:
|
|
396
|
-
reason = exc.reason if exc.reason else str(exc)
|
|
397
|
-
raise RuntimeError(str(reason)) from exc
|
|
398
|
-
except OSError as exc:
|
|
399
|
-
raise RuntimeError(str(exc)) from exc
|
|
400
|
-
|
|
401
|
-
if not raw:
|
|
402
|
-
return {}
|
|
403
|
-
|
|
404
|
-
try:
|
|
405
|
-
parsed = json.loads(raw.decode("utf-8"))
|
|
406
|
-
except json.JSONDecodeError as exc:
|
|
407
|
-
raise RuntimeError("Invalid JSON response from stream API") from exc
|
|
408
|
-
|
|
409
|
-
if isinstance(parsed, dict):
|
|
410
|
-
return parsed
|
|
411
|
-
raise RuntimeError("Unexpected stream API response payload")
|
|
412
|
-
|
|
413
375
|
def _trim_pending(self) -> None:
|
|
414
376
|
overflow = len(self._pending) - self.config.max_pending_events
|
|
415
377
|
if overflow <= 0:
|
|
@@ -459,6 +421,112 @@ def coerce_output_stream(spec: Any) -> OutputStream | None:
|
|
|
459
421
|
)
|
|
460
422
|
|
|
461
423
|
|
|
424
|
+
def check_stream_handshake(
|
|
425
|
+
config: EulerViewStreamConfig | Mapping[str, Any],
|
|
426
|
+
*,
|
|
427
|
+
run_id: str | None = None,
|
|
428
|
+
stream_attach_token: str | None = None,
|
|
429
|
+
slurm_job_id: int | None = None,
|
|
430
|
+
datasource_id: int | None = None,
|
|
431
|
+
euler_train_dir: str | None = None,
|
|
432
|
+
run_dir: str | None = None,
|
|
433
|
+
) -> dict[str, Any]:
|
|
434
|
+
if isinstance(config, Mapping):
|
|
435
|
+
resolved_config = _config_from_mapping(config)
|
|
436
|
+
elif isinstance(config, EulerViewStreamConfig):
|
|
437
|
+
resolved_config = config
|
|
438
|
+
else:
|
|
439
|
+
raise TypeError(
|
|
440
|
+
"config must be a mapping or EulerViewStreamConfig",
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
if resolved_config.stream_token is not None:
|
|
444
|
+
raise ValueError(
|
|
445
|
+
"stream.stream_token bypasses the session handshake; a dry-run handshake requires api_token or access_token instead",
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
normalized_run_id = (
|
|
449
|
+
_normalize_optional_text(run_id, field_name="run_id")
|
|
450
|
+
if run_id is not None
|
|
451
|
+
else _default_stream_check_run_id()
|
|
452
|
+
)
|
|
453
|
+
if normalized_run_id is None:
|
|
454
|
+
normalized_run_id = _default_stream_check_run_id()
|
|
455
|
+
normalized_stream_attach_token = (
|
|
456
|
+
_normalize_optional_text(
|
|
457
|
+
stream_attach_token,
|
|
458
|
+
field_name="stream_attach_token",
|
|
459
|
+
)
|
|
460
|
+
if stream_attach_token is not None
|
|
461
|
+
else None
|
|
462
|
+
)
|
|
463
|
+
if (
|
|
464
|
+
normalized_stream_attach_token is not None
|
|
465
|
+
and resolved_config.stream_attach_token is not None
|
|
466
|
+
and normalized_stream_attach_token != resolved_config.stream_attach_token
|
|
467
|
+
):
|
|
468
|
+
raise ValueError(
|
|
469
|
+
"stream_attach_token argument does not match stream.stream_attach_token",
|
|
470
|
+
)
|
|
471
|
+
resolved_stream_attach_token = (
|
|
472
|
+
normalized_stream_attach_token or resolved_config.stream_attach_token
|
|
473
|
+
)
|
|
474
|
+
if resolved_config.model_id is None and resolved_stream_attach_token is None:
|
|
475
|
+
raise ValueError(
|
|
476
|
+
"stream.model_id is required for handshake checks when stream_attach_token is not provided",
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
resolved_datasource_id = (
|
|
480
|
+
_require_positive_int(
|
|
481
|
+
datasource_id,
|
|
482
|
+
field_name="datasource_id",
|
|
483
|
+
allow_none=True,
|
|
484
|
+
)
|
|
485
|
+
if datasource_id is not None
|
|
486
|
+
else resolved_config.datasource_id
|
|
487
|
+
)
|
|
488
|
+
resolved_slurm_job_id = (
|
|
489
|
+
_require_positive_int(
|
|
490
|
+
slurm_job_id,
|
|
491
|
+
field_name="slurm_job_id",
|
|
492
|
+
allow_none=True,
|
|
493
|
+
)
|
|
494
|
+
if slurm_job_id is not None
|
|
495
|
+
else None
|
|
496
|
+
)
|
|
497
|
+
resolved_euler_train_dir = (
|
|
498
|
+
_normalize_optional_text(
|
|
499
|
+
euler_train_dir,
|
|
500
|
+
field_name="euler_train_dir",
|
|
501
|
+
)
|
|
502
|
+
if euler_train_dir is not None
|
|
503
|
+
else resolved_config.euler_train_dir
|
|
504
|
+
)
|
|
505
|
+
resolved_run_dir = (
|
|
506
|
+
_normalize_optional_text(run_dir, field_name="run_dir")
|
|
507
|
+
if run_dir is not None
|
|
508
|
+
else resolved_config.run_dir
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
url = f"{resolved_config.base_url}/api/model-run-stream/check"
|
|
512
|
+
payload = _build_stream_session_body(
|
|
513
|
+
run_id=normalized_run_id,
|
|
514
|
+
model_id=resolved_config.model_id,
|
|
515
|
+
run_dir=resolved_run_dir,
|
|
516
|
+
euler_train_dir=resolved_euler_train_dir,
|
|
517
|
+
stream_attach_token=resolved_stream_attach_token,
|
|
518
|
+
slurm_job_id=resolved_slurm_job_id,
|
|
519
|
+
datasource_id=resolved_datasource_id,
|
|
520
|
+
session_expires_in_sec=resolved_config.session_expires_in_sec,
|
|
521
|
+
)
|
|
522
|
+
return _post_json_request(
|
|
523
|
+
url,
|
|
524
|
+
payload,
|
|
525
|
+
token=resolved_config.api_token or resolved_config.access_token,
|
|
526
|
+
timeout_sec=resolved_config.timeout_sec,
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
|
|
462
530
|
def _looks_like_consumer(value: Any) -> bool:
|
|
463
531
|
return all(
|
|
464
532
|
callable(getattr(value, name, None))
|
|
@@ -562,6 +630,40 @@ def _extract_session_stream_attach_token(payload: Mapping[str, Any]) -> str | No
|
|
|
562
630
|
return normalized or None
|
|
563
631
|
|
|
564
632
|
|
|
633
|
+
def _build_stream_session_body(
|
|
634
|
+
*,
|
|
635
|
+
run_id: str,
|
|
636
|
+
model_id: int | None,
|
|
637
|
+
run_dir: str | None,
|
|
638
|
+
euler_train_dir: str | None,
|
|
639
|
+
stream_attach_token: str | None,
|
|
640
|
+
slurm_job_id: int | None,
|
|
641
|
+
datasource_id: int | None,
|
|
642
|
+
session_expires_in_sec: int | None,
|
|
643
|
+
) -> dict[str, Any]:
|
|
644
|
+
body: dict[str, Any] = {"runId": run_id}
|
|
645
|
+
if model_id is not None:
|
|
646
|
+
body["modelId"] = model_id
|
|
647
|
+
if run_dir is not None:
|
|
648
|
+
body["runDir"] = run_dir
|
|
649
|
+
if euler_train_dir is not None:
|
|
650
|
+
body["eulerTrainDir"] = euler_train_dir
|
|
651
|
+
if stream_attach_token is not None:
|
|
652
|
+
body["streamAttachToken"] = stream_attach_token
|
|
653
|
+
if slurm_job_id is not None:
|
|
654
|
+
body["slurmJobId"] = slurm_job_id
|
|
655
|
+
if datasource_id is not None:
|
|
656
|
+
body["datasourceId"] = datasource_id
|
|
657
|
+
if session_expires_in_sec is not None:
|
|
658
|
+
body["expiresInSec"] = session_expires_in_sec
|
|
659
|
+
return body
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _default_stream_check_run_id() -> str:
|
|
663
|
+
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
|
664
|
+
return f"stream-check-{timestamp}"
|
|
665
|
+
|
|
666
|
+
|
|
565
667
|
def _normalize_optional_text(
|
|
566
668
|
value: Any,
|
|
567
669
|
*,
|
|
@@ -636,6 +738,52 @@ def _extract_http_error_message(error: HTTPError) -> str:
|
|
|
636
738
|
return f"HTTP {error.code}: {error.reason}"
|
|
637
739
|
|
|
638
740
|
|
|
741
|
+
def _post_json_request(
|
|
742
|
+
url: str,
|
|
743
|
+
payload: dict[str, Any],
|
|
744
|
+
*,
|
|
745
|
+
token: str | None,
|
|
746
|
+
timeout_sec: float,
|
|
747
|
+
) -> dict[str, Any]:
|
|
748
|
+
headers = {
|
|
749
|
+
"Accept": "application/json",
|
|
750
|
+
"Content-Type": "application/json",
|
|
751
|
+
}
|
|
752
|
+
if token:
|
|
753
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
754
|
+
|
|
755
|
+
request = Request(
|
|
756
|
+
url,
|
|
757
|
+
data=json.dumps(payload, default=_json_default).encode("utf-8"),
|
|
758
|
+
headers=headers,
|
|
759
|
+
method="POST",
|
|
760
|
+
)
|
|
761
|
+
|
|
762
|
+
try:
|
|
763
|
+
with urlopen(request, timeout=timeout_sec) as response:
|
|
764
|
+
raw = response.read()
|
|
765
|
+
except HTTPError as exc:
|
|
766
|
+
message = _extract_http_error_message(exc)
|
|
767
|
+
raise RuntimeError(message) from exc
|
|
768
|
+
except URLError as exc:
|
|
769
|
+
reason = exc.reason if exc.reason else str(exc)
|
|
770
|
+
raise RuntimeError(str(reason)) from exc
|
|
771
|
+
except OSError as exc:
|
|
772
|
+
raise RuntimeError(str(exc)) from exc
|
|
773
|
+
|
|
774
|
+
if not raw:
|
|
775
|
+
return {}
|
|
776
|
+
|
|
777
|
+
try:
|
|
778
|
+
parsed = json.loads(raw.decode("utf-8"))
|
|
779
|
+
except json.JSONDecodeError as exc:
|
|
780
|
+
raise RuntimeError("Invalid JSON response from stream API") from exc
|
|
781
|
+
|
|
782
|
+
if isinstance(parsed, dict):
|
|
783
|
+
return parsed
|
|
784
|
+
raise RuntimeError("Unexpected stream API response payload")
|
|
785
|
+
|
|
786
|
+
|
|
639
787
|
def _to_jsonable(value: Any) -> Any:
|
|
640
788
|
return json.loads(json.dumps(value, default=_json_default))
|
|
641
789
|
|
|
@@ -646,5 +794,6 @@ __all__ = [
|
|
|
646
794
|
"OutputStream",
|
|
647
795
|
"OutputStreamConsumer",
|
|
648
796
|
"StreamContext",
|
|
797
|
+
"check_stream_handshake",
|
|
649
798
|
"coerce_output_stream",
|
|
650
799
|
]
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
from .stream import check_stream_handshake
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
12
|
+
parser = argparse.ArgumentParser(
|
|
13
|
+
prog="euler-train.stream.check",
|
|
14
|
+
description="Dry-run the Euler View stream handshake",
|
|
15
|
+
)
|
|
16
|
+
parser.add_argument("--api-url", required=True, help="Euler View base URL")
|
|
17
|
+
parser.add_argument(
|
|
18
|
+
"--model-id",
|
|
19
|
+
type=int,
|
|
20
|
+
help="Euler View model ID. Required when using the SLURM fallback path instead of --stream-attach-token.",
|
|
21
|
+
)
|
|
22
|
+
parser.add_argument(
|
|
23
|
+
"--api-key",
|
|
24
|
+
required=True,
|
|
25
|
+
help="Euler View API token used to authorize the handshake check",
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
"--run-id",
|
|
29
|
+
help="Run ID to use for the dry-run handshake. Defaults to an ephemeral stream-check ID.",
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--stream-attach-token",
|
|
33
|
+
help="Optional explicit launch attachment token to validate",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--slurm-job-id",
|
|
37
|
+
type=int,
|
|
38
|
+
help="Optional SLURM job ID to test the fallback matching path",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument(
|
|
41
|
+
"--datasource-id",
|
|
42
|
+
type=int,
|
|
43
|
+
help="Optional datasource hint to include in the handshake payload",
|
|
44
|
+
)
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"--euler-train-dir",
|
|
47
|
+
help="Optional euler_train runs directory path to include in the handshake payload",
|
|
48
|
+
)
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--run-dir",
|
|
51
|
+
help="Optional run directory path to include in the handshake payload",
|
|
52
|
+
)
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--timeout-sec",
|
|
55
|
+
type=float,
|
|
56
|
+
default=10.0,
|
|
57
|
+
help="HTTP timeout in seconds (default: 10)",
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument(
|
|
60
|
+
"--json",
|
|
61
|
+
action="store_true",
|
|
62
|
+
help="Print the raw JSON response instead of a human-readable summary",
|
|
63
|
+
)
|
|
64
|
+
return parser
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
68
|
+
parser = build_parser()
|
|
69
|
+
args = parser.parse_args(argv)
|
|
70
|
+
|
|
71
|
+
if args.model_id is None and not args.stream_attach_token:
|
|
72
|
+
parser.error("--model-id is required unless --stream-attach-token is provided")
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
result = check_stream_handshake(
|
|
76
|
+
{
|
|
77
|
+
"base_url": args.api_url,
|
|
78
|
+
"model_id": args.model_id,
|
|
79
|
+
"api_token": args.api_key,
|
|
80
|
+
"stream_attach_token": args.stream_attach_token,
|
|
81
|
+
"datasource_id": args.datasource_id,
|
|
82
|
+
"euler_train_dir": args.euler_train_dir,
|
|
83
|
+
"run_dir": args.run_dir,
|
|
84
|
+
"timeout_sec": args.timeout_sec,
|
|
85
|
+
},
|
|
86
|
+
run_id=args.run_id,
|
|
87
|
+
slurm_job_id=args.slurm_job_id,
|
|
88
|
+
)
|
|
89
|
+
except Exception as exc:
|
|
90
|
+
if args.json:
|
|
91
|
+
print(json.dumps({"success": False, "error": str(exc)}, indent=2))
|
|
92
|
+
else:
|
|
93
|
+
print(f"Handshake check failed: {exc}", file=sys.stderr)
|
|
94
|
+
return 1
|
|
95
|
+
|
|
96
|
+
if args.json:
|
|
97
|
+
print(json.dumps(result, indent=2, sort_keys=True))
|
|
98
|
+
return 0
|
|
99
|
+
|
|
100
|
+
run = result.get("run", {})
|
|
101
|
+
print("Handshake check succeeded")
|
|
102
|
+
print(f" model_id: {run.get('modelId')}")
|
|
103
|
+
print(f" run_id: {run.get('runId')}")
|
|
104
|
+
print(f" resolution: {result.get('resolution')}")
|
|
105
|
+
print(f" stream_attach_token: {run.get('streamAttachToken')}")
|
|
106
|
+
print(f" ingest_url: {result.get('ingestUrl')}")
|
|
107
|
+
return 0
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
if __name__ == "__main__":
|
|
111
|
+
raise SystemExit(main())
|
|
@@ -10,9 +10,11 @@ euler_train/run.py
|
|
|
10
10
|
euler_train/serialization.py
|
|
11
11
|
euler_train/slurm.py
|
|
12
12
|
euler_train/stream.py
|
|
13
|
+
euler_train/stream_check_cli.py
|
|
13
14
|
euler_train.egg-info/PKG-INFO
|
|
14
15
|
euler_train.egg-info/SOURCES.txt
|
|
15
16
|
euler_train.egg-info/dependency_links.txt
|
|
17
|
+
euler_train.egg-info/entry_points.txt
|
|
16
18
|
euler_train.egg-info/requires.txt
|
|
17
19
|
euler_train.egg-info/top_level.txt
|
|
18
20
|
tests/test_runlog.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "euler-train"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.9.0"
|
|
8
8
|
description = "Lightweight file-based experiment logger for PyTorch"
|
|
9
9
|
requires-python = ">=3.9"
|
|
10
10
|
dependencies = ["numpy", "euler-dataset-contract"]
|
|
@@ -16,5 +16,8 @@ architecture = ["onnx", "onnxruntime", "onnxsim"]
|
|
|
16
16
|
naming = ["euler-metric-naming"]
|
|
17
17
|
dev = ["pytest", "Pillow", "torch", "nvidia-ml-py", "onnx", "onnxruntime", "onnxsim"]
|
|
18
18
|
|
|
19
|
+
[project.scripts]
|
|
20
|
+
"euler-train.stream.check" = "euler_train.stream_check_cli:main"
|
|
21
|
+
|
|
19
22
|
[tool.pytest.ini_options]
|
|
20
23
|
pythonpath = ["."]
|
|
@@ -16,6 +16,7 @@ from PIL import Image
|
|
|
16
16
|
|
|
17
17
|
import euler_train
|
|
18
18
|
import euler_train.stream as stream_mod
|
|
19
|
+
from euler_train.stream_check_cli import main as stream_check_cli_main
|
|
19
20
|
|
|
20
21
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
21
22
|
# Helpers
|
|
@@ -1962,7 +1963,7 @@ class TestStreaming:
|
|
|
1962
1963
|
"timeout": timeout,
|
|
1963
1964
|
},
|
|
1964
1965
|
)
|
|
1965
|
-
if request.full_url.
|
|
1966
|
+
if request.full_url == "https://sync.example/api/model-run-stream/session":
|
|
1966
1967
|
return _FakeHttpResponse(
|
|
1967
1968
|
{
|
|
1968
1969
|
"token": "stream-token",
|
|
@@ -1996,12 +1997,12 @@ class TestStreaming:
|
|
|
1996
1997
|
run.log({"loss": 0.25}, step=3, epoch=1)
|
|
1997
1998
|
run.finish()
|
|
1998
1999
|
|
|
1999
|
-
assert calls[0]["url"] ==
|
|
2000
|
-
f"https://sync.example/api/models/42/runs/{run.run_id}/stream/session"
|
|
2001
|
-
)
|
|
2000
|
+
assert calls[0]["url"] == "https://sync.example/api/model-run-stream/session"
|
|
2002
2001
|
assert calls[0]["method"] == "POST"
|
|
2003
2002
|
assert calls[0]["headers"]["authorization"] == "Bearer user-token"
|
|
2004
2003
|
assert calls[0]["body"] == {
|
|
2004
|
+
"runId": run.run_id,
|
|
2005
|
+
"modelId": 42,
|
|
2005
2006
|
"runDir": str(run.dir),
|
|
2006
2007
|
"eulerTrainDir": str(run.dir.parent),
|
|
2007
2008
|
"datasourceId": 7,
|
|
@@ -2027,6 +2028,73 @@ class TestStreaming:
|
|
|
2027
2028
|
finish_event = ingest_calls[2]["body"]["events"][0]
|
|
2028
2029
|
assert finish_event["status"] == "completed"
|
|
2029
2030
|
|
|
2031
|
+
def test_euler_view_http_consumer_can_handshake_without_model_id_when_attach_token_is_present(
|
|
2032
|
+
self,
|
|
2033
|
+
tmp_path,
|
|
2034
|
+
monkeypatch,
|
|
2035
|
+
):
|
|
2036
|
+
calls: list[dict] = []
|
|
2037
|
+
|
|
2038
|
+
def fake_urlopen(request, timeout=0):
|
|
2039
|
+
headers = {k.lower(): v for k, v in request.header_items()}
|
|
2040
|
+
body = json.loads(request.data.decode("utf-8"))
|
|
2041
|
+
calls.append(
|
|
2042
|
+
{
|
|
2043
|
+
"url": request.full_url,
|
|
2044
|
+
"method": request.get_method(),
|
|
2045
|
+
"headers": headers,
|
|
2046
|
+
"body": body,
|
|
2047
|
+
"timeout": timeout,
|
|
2048
|
+
},
|
|
2049
|
+
)
|
|
2050
|
+
if request.full_url.endswith("/api/model-run-stream/session"):
|
|
2051
|
+
return _FakeHttpResponse(
|
|
2052
|
+
{
|
|
2053
|
+
"token": "stream-token",
|
|
2054
|
+
"expiresAt": "2099-01-01T00:00:00+00:00",
|
|
2055
|
+
"ingestUrl": "https://sync.example/api/model-run-stream/ingest",
|
|
2056
|
+
"run": {
|
|
2057
|
+
"modelId": 42,
|
|
2058
|
+
"runId": "placeholder",
|
|
2059
|
+
"streamAttachToken": "attach-123",
|
|
2060
|
+
},
|
|
2061
|
+
},
|
|
2062
|
+
)
|
|
2063
|
+
if request.full_url.endswith("/api/model-run-stream/ingest"):
|
|
2064
|
+
return _FakeHttpResponse(
|
|
2065
|
+
{
|
|
2066
|
+
"success": True,
|
|
2067
|
+
"events": [],
|
|
2068
|
+
"latestCursor": len(calls),
|
|
2069
|
+
},
|
|
2070
|
+
)
|
|
2071
|
+
raise AssertionError(f"Unexpected URL {request.full_url}")
|
|
2072
|
+
|
|
2073
|
+
monkeypatch.setattr(stream_mod, "urlopen", fake_urlopen)
|
|
2074
|
+
|
|
2075
|
+
run = euler_train.init(
|
|
2076
|
+
dir=str(tmp_path / "r"),
|
|
2077
|
+
config={"lr": 1e-3},
|
|
2078
|
+
stream={
|
|
2079
|
+
"base_url": "https://sync.example",
|
|
2080
|
+
"api_token": "user-token",
|
|
2081
|
+
"stream_attach_token": "attach-123",
|
|
2082
|
+
"batch_size": 1,
|
|
2083
|
+
},
|
|
2084
|
+
)
|
|
2085
|
+
run.log({"loss": 0.25}, step=3, epoch=1)
|
|
2086
|
+
run.finish()
|
|
2087
|
+
|
|
2088
|
+
assert calls[0]["url"] == "https://sync.example/api/model-run-stream/session"
|
|
2089
|
+
assert calls[0]["method"] == "POST"
|
|
2090
|
+
assert calls[0]["headers"]["authorization"] == "Bearer user-token"
|
|
2091
|
+
assert calls[0]["body"] == {
|
|
2092
|
+
"runId": run.run_id,
|
|
2093
|
+
"runDir": str(run.dir),
|
|
2094
|
+
"eulerTrainDir": str(run.dir.parent),
|
|
2095
|
+
"streamAttachToken": "attach-123",
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2030
2098
|
def test_stream_network_failures_do_not_break_training(
|
|
2031
2099
|
self,
|
|
2032
2100
|
tmp_path,
|
|
@@ -2053,6 +2121,121 @@ class TestStreaming:
|
|
|
2053
2121
|
meta = _read_json(run.dir / "meta.json")
|
|
2054
2122
|
assert meta["status"] == "completed"
|
|
2055
2123
|
|
|
2124
|
+
def test_check_stream_handshake_uses_dry_run_route(
|
|
2125
|
+
self,
|
|
2126
|
+
monkeypatch,
|
|
2127
|
+
):
|
|
2128
|
+
calls: list[dict] = []
|
|
2129
|
+
|
|
2130
|
+
def fake_urlopen(request, timeout=0):
|
|
2131
|
+
headers = {k.lower(): v for k, v in request.header_items()}
|
|
2132
|
+
body = json.loads(request.data.decode("utf-8"))
|
|
2133
|
+
calls.append(
|
|
2134
|
+
{
|
|
2135
|
+
"url": request.full_url,
|
|
2136
|
+
"method": request.get_method(),
|
|
2137
|
+
"headers": headers,
|
|
2138
|
+
"body": body,
|
|
2139
|
+
"timeout": timeout,
|
|
2140
|
+
},
|
|
2141
|
+
)
|
|
2142
|
+
if request.full_url == "https://sync.example/api/model-run-stream/check":
|
|
2143
|
+
return _FakeHttpResponse(
|
|
2144
|
+
{
|
|
2145
|
+
"success": True,
|
|
2146
|
+
"resolution": "stream_attach_token",
|
|
2147
|
+
"ingestUrl": "https://sync.example/api/model-run-stream/ingest",
|
|
2148
|
+
"run": {
|
|
2149
|
+
"modelId": 42,
|
|
2150
|
+
"runId": "dry-run-1",
|
|
2151
|
+
"streamAttachToken": "attach-123",
|
|
2152
|
+
"launchId": "launch-abc",
|
|
2153
|
+
"datasourceId": 7,
|
|
2154
|
+
"eulerTrainDir": "/outputs",
|
|
2155
|
+
"runDir": "/outputs/dry-run-1",
|
|
2156
|
+
},
|
|
2157
|
+
},
|
|
2158
|
+
)
|
|
2159
|
+
raise AssertionError(f"Unexpected URL {request.full_url}")
|
|
2160
|
+
|
|
2161
|
+
monkeypatch.setattr(stream_mod, "urlopen", fake_urlopen)
|
|
2162
|
+
|
|
2163
|
+
payload = stream_mod.check_stream_handshake(
|
|
2164
|
+
{
|
|
2165
|
+
"base_url": "https://sync.example",
|
|
2166
|
+
"api_token": "user-token",
|
|
2167
|
+
"stream_attach_token": "attach-123",
|
|
2168
|
+
"datasource_id": 7,
|
|
2169
|
+
},
|
|
2170
|
+
run_id="dry-run-1",
|
|
2171
|
+
)
|
|
2172
|
+
|
|
2173
|
+
assert payload["success"] is True
|
|
2174
|
+
assert calls == [
|
|
2175
|
+
{
|
|
2176
|
+
"url": "https://sync.example/api/model-run-stream/check",
|
|
2177
|
+
"method": "POST",
|
|
2178
|
+
"headers": {
|
|
2179
|
+
"accept": "application/json",
|
|
2180
|
+
"authorization": "Bearer user-token",
|
|
2181
|
+
"content-type": "application/json",
|
|
2182
|
+
},
|
|
2183
|
+
"body": {
|
|
2184
|
+
"runId": "dry-run-1",
|
|
2185
|
+
"streamAttachToken": "attach-123",
|
|
2186
|
+
"datasourceId": 7,
|
|
2187
|
+
},
|
|
2188
|
+
"timeout": 10.0,
|
|
2189
|
+
},
|
|
2190
|
+
]
|
|
2191
|
+
|
|
2192
|
+
def test_stream_check_cli_prints_json_payload(
|
|
2193
|
+
self,
|
|
2194
|
+
monkeypatch,
|
|
2195
|
+
capsys,
|
|
2196
|
+
):
|
|
2197
|
+
def fake_urlopen(request, timeout=0):
|
|
2198
|
+
if request.full_url != "https://sync.example/api/model-run-stream/check":
|
|
2199
|
+
raise AssertionError(f"Unexpected URL {request.full_url}")
|
|
2200
|
+
return _FakeHttpResponse(
|
|
2201
|
+
{
|
|
2202
|
+
"success": True,
|
|
2203
|
+
"resolution": "unresolved",
|
|
2204
|
+
"ingestUrl": "https://sync.example/api/model-run-stream/ingest",
|
|
2205
|
+
"run": {
|
|
2206
|
+
"modelId": 42,
|
|
2207
|
+
"runId": "dry-run-2",
|
|
2208
|
+
"streamAttachToken": None,
|
|
2209
|
+
"launchId": None,
|
|
2210
|
+
"datasourceId": None,
|
|
2211
|
+
"eulerTrainDir": None,
|
|
2212
|
+
"runDir": None,
|
|
2213
|
+
},
|
|
2214
|
+
},
|
|
2215
|
+
)
|
|
2216
|
+
|
|
2217
|
+
monkeypatch.setattr(stream_mod, "urlopen", fake_urlopen)
|
|
2218
|
+
|
|
2219
|
+
exit_code = stream_check_cli_main(
|
|
2220
|
+
[
|
|
2221
|
+
"--api-url",
|
|
2222
|
+
"https://sync.example",
|
|
2223
|
+
"--api-key",
|
|
2224
|
+
"user-token",
|
|
2225
|
+
"--run-id",
|
|
2226
|
+
"dry-run-2",
|
|
2227
|
+
"--stream-attach-token",
|
|
2228
|
+
"attach-123",
|
|
2229
|
+
"--json",
|
|
2230
|
+
],
|
|
2231
|
+
)
|
|
2232
|
+
|
|
2233
|
+
assert exit_code == 0
|
|
2234
|
+
payload = json.loads(capsys.readouterr().out)
|
|
2235
|
+
assert payload["success"] is True
|
|
2236
|
+
assert payload["resolution"] == "unresolved"
|
|
2237
|
+
assert payload["run"]["runId"] == "dry-run-2"
|
|
2238
|
+
|
|
2056
2239
|
|
|
2057
2240
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
2058
2241
|
# lifecycle (finish, context manager, crash)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|