gcube-cli 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/PKG-INFO +32 -1
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/README.md +31 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/pyproject.toml +1 -1
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/sse.py +20 -2
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/workload.py +14 -1
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/workload.py +293 -32
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli.egg-info/PKG-INFO +32 -1
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload.py +8 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload_commands.py +27 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload_sse.py +213 -2
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/setup.cfg +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/__init__.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/__main__.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/_group.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/__init__.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/client.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/credential.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/logs.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/point.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/resource.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/storage.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/api/user.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/cli.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/__init__.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/configure.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/credential.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/gpu.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/point.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/resource.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/commands/storage.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/config/__init__.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/config/config.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/output/__init__.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/output/json_out.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/output/table.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/output/yaml_out.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli/update_check.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli.egg-info/SOURCES.txt +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli.egg-info/dependency_links.txt +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli.egg-info/entry_points.txt +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli.egg-info/requires.txt +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/src/gcube_cli.egg-info/top_level.txt +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_api_client.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_cli.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_configure.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_credential.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_gpu.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_output.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_point.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_resource.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_update_check.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_user_api.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload_hints.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload_logs.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload_register.py +0 -0
- {gcube_cli-0.2.0 → gcube_cli-0.3.0}/tests/test_workload_update.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gcube-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Official command-line tool for the gcube AI GPU cloud platform
|
|
5
5
|
Author-email: Data Alliance <support@data-alliance.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -162,7 +162,9 @@ Manage the full lifecycle of GPU workloads. Each workload is identified by a **S
|
|
|
162
162
|
| `gcube workload describe <ser>` | Show workload details |
|
|
163
163
|
| `gcube workload start <ser>` | Start a workload and monitor deployment status in real time |
|
|
164
164
|
| `gcube workload start <ser> --no-watch` | Start a workload without monitoring |
|
|
165
|
+
| `gcube workload start <ser> --timeout <sec>` | Stop monitoring after N seconds if the deploy hasn't finished (exit code 124) |
|
|
165
166
|
| `gcube workload watch <ser>` | Re-attach to a deploying workload and monitor its status |
|
|
167
|
+
| `gcube workload watch <ser> --plain` | Force plain line-per-event output instead of the live dashboard |
|
|
166
168
|
| `gcube workload stop <ser>` | Stop a workload |
|
|
167
169
|
| `gcube workload delete <ser>` | Delete a workload |
|
|
168
170
|
| `gcube workload logs <ser>` | Stream container logs in real time |
|
|
@@ -248,6 +250,14 @@ gpuSpecs:
|
|
|
248
250
|
- Use `--no-watch` to start without monitoring (e.g. in scripts).
|
|
249
251
|
- For deployments with more than 5 replicas, progress is shown as an aggregate counter (e.g. `Deploying... 60/100`) instead of per-stage detail.
|
|
250
252
|
|
|
253
|
+
**Non-interactive / scripting**
|
|
254
|
+
|
|
255
|
+
`start` and `watch` adapt their output for automation:
|
|
256
|
+
|
|
257
|
+
- When stdout is **not a terminal** (piped or redirected), or with `--plain`, output switches from the live dashboard to a plain line-per-event stream (`HH:MM:SS [ser] message`, server timestamps, flushed per line — safe to `tail -f` and durable even if the process is hard-killed).
|
|
258
|
+
- With `-o json` / `-o yaml`, only a single result object is printed: `{ser, status, startedAt, completedAt, durationSec, failReason}`. `status` is one of `success`, `failed`, `timeout`, `interrupted`, `skipped`.
|
|
259
|
+
- `--timeout <sec>` stops an unfinished watch after N seconds and exits `124` (GNU `timeout` convention); it also keeps `Ctrl+C` responsive on a silent stream.
|
|
260
|
+
|
|
251
261
|
---
|
|
252
262
|
|
|
253
263
|
### GPU
|
|
@@ -355,6 +365,7 @@ gcube -o yaml workload list # YAML
|
|
|
355
365
|
| `2` | gcube API error |
|
|
356
366
|
| `3` | Authentication failure or token expired |
|
|
357
367
|
| `4` | Network error |
|
|
368
|
+
| `124` | `workload watch`/`start` timed out (`--timeout`) before the deploy finished |
|
|
358
369
|
|
|
359
370
|
---
|
|
360
371
|
|
|
@@ -394,6 +405,26 @@ gcube workload stop 2212
|
|
|
394
405
|
gcube workload delete 2212
|
|
395
406
|
```
|
|
396
407
|
|
|
408
|
+
### Scripting / Batch Deployment
|
|
409
|
+
|
|
410
|
+
```bash
|
|
411
|
+
# Plain log to a file (auto-detected; no live dashboard). Capture stderr too so
|
|
412
|
+
# failure/timeout messages land in the log. Branch on the exit code.
|
|
413
|
+
gcube workload watch 2212 --timeout 1800 > 2212.log 2>&1
|
|
414
|
+
echo "exit=$?" # 0 success · 1 failed · 124 timeout
|
|
415
|
+
|
|
416
|
+
# Machine-readable result for aggregating many deploys
|
|
417
|
+
gcube -o json workload watch 2212 --timeout 1800 > 2212.json
|
|
418
|
+
# {"ser":2212,"status":"success","startedAt":...,"completedAt":...,"durationSec":...,"failReason":null}
|
|
419
|
+
|
|
420
|
+
# Per-student burst with distinct tokens (one process each)
|
|
421
|
+
for ser in "${SERS[@]}"; do
|
|
422
|
+
GCUBE_ACCESS_TOKEN="${TOKENS[$ser]}" \
|
|
423
|
+
gcube -o json workload watch "$ser" --timeout 1800 > "$ser.json" &
|
|
424
|
+
done
|
|
425
|
+
wait
|
|
426
|
+
```
|
|
427
|
+
|
|
397
428
|
### Update a Workload
|
|
398
429
|
|
|
399
430
|
```bash
|
|
@@ -123,7 +123,9 @@ Manage the full lifecycle of GPU workloads. Each workload is identified by a **S
|
|
|
123
123
|
| `gcube workload describe <ser>` | Show workload details |
|
|
124
124
|
| `gcube workload start <ser>` | Start a workload and monitor deployment status in real time |
|
|
125
125
|
| `gcube workload start <ser> --no-watch` | Start a workload without monitoring |
|
|
126
|
+
| `gcube workload start <ser> --timeout <sec>` | Stop monitoring after N seconds if the deploy hasn't finished (exit code 124) |
|
|
126
127
|
| `gcube workload watch <ser>` | Re-attach to a deploying workload and monitor its status |
|
|
128
|
+
| `gcube workload watch <ser> --plain` | Force plain line-per-event output instead of the live dashboard |
|
|
127
129
|
| `gcube workload stop <ser>` | Stop a workload |
|
|
128
130
|
| `gcube workload delete <ser>` | Delete a workload |
|
|
129
131
|
| `gcube workload logs <ser>` | Stream container logs in real time |
|
|
@@ -209,6 +211,14 @@ gpuSpecs:
|
|
|
209
211
|
- Use `--no-watch` to start without monitoring (e.g. in scripts).
|
|
210
212
|
- For deployments with more than 5 replicas, progress is shown as an aggregate counter (e.g. `Deploying... 60/100`) instead of per-stage detail.
|
|
211
213
|
|
|
214
|
+
**Non-interactive / scripting**
|
|
215
|
+
|
|
216
|
+
`start` and `watch` adapt their output for automation:
|
|
217
|
+
|
|
218
|
+
- When stdout is **not a terminal** (piped or redirected), or with `--plain`, output switches from the live dashboard to a plain line-per-event stream (`HH:MM:SS [ser] message`, server timestamps, flushed per line — safe to `tail -f` and durable even if the process is hard-killed).
|
|
219
|
+
- With `-o json` / `-o yaml`, only a single result object is printed: `{ser, status, startedAt, completedAt, durationSec, failReason}`. `status` is one of `success`, `failed`, `timeout`, `interrupted`, `skipped`.
|
|
220
|
+
- `--timeout <sec>` stops an unfinished watch after N seconds and exits `124` (GNU `timeout` convention); it also keeps `Ctrl+C` responsive on a silent stream.
|
|
221
|
+
|
|
212
222
|
---
|
|
213
223
|
|
|
214
224
|
### GPU
|
|
@@ -316,6 +326,7 @@ gcube -o yaml workload list # YAML
|
|
|
316
326
|
| `2` | gcube API error |
|
|
317
327
|
| `3` | Authentication failure or token expired |
|
|
318
328
|
| `4` | Network error |
|
|
329
|
+
| `124` | `workload watch`/`start` timed out (`--timeout`) before the deploy finished |
|
|
319
330
|
|
|
320
331
|
---
|
|
321
332
|
|
|
@@ -355,6 +366,26 @@ gcube workload stop 2212
|
|
|
355
366
|
gcube workload delete 2212
|
|
356
367
|
```
|
|
357
368
|
|
|
369
|
+
### Scripting / Batch Deployment
|
|
370
|
+
|
|
371
|
+
```bash
|
|
372
|
+
# Plain log to a file (auto-detected; no live dashboard). Capture stderr too so
|
|
373
|
+
# failure/timeout messages land in the log. Branch on the exit code.
|
|
374
|
+
gcube workload watch 2212 --timeout 1800 > 2212.log 2>&1
|
|
375
|
+
echo "exit=$?" # 0 success · 1 failed · 124 timeout
|
|
376
|
+
|
|
377
|
+
# Machine-readable result for aggregating many deploys
|
|
378
|
+
gcube -o json workload watch 2212 --timeout 1800 > 2212.json
|
|
379
|
+
# {"ser":2212,"status":"success","startedAt":...,"completedAt":...,"durationSec":...,"failReason":null}
|
|
380
|
+
|
|
381
|
+
# Per-student burst with distinct tokens (one process each)
|
|
382
|
+
for ser in "${SERS[@]}"; do
|
|
383
|
+
GCUBE_ACCESS_TOKEN="${TOKENS[$ser]}" \
|
|
384
|
+
gcube -o json workload watch "$ser" --timeout 1800 > "$ser.json" &
|
|
385
|
+
done
|
|
386
|
+
wait
|
|
387
|
+
```
|
|
388
|
+
|
|
358
389
|
### Update a Workload
|
|
359
390
|
|
|
360
391
|
```bash
|
|
@@ -23,12 +23,28 @@ class SSEAuthError(Exception):
|
|
|
23
23
|
pass
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
class SSEReadTimeout(Exception):
|
|
27
|
+
"""No data received within ``read_timeout`` — the stream went idle (not a failure).
|
|
28
|
+
|
|
29
|
+
Kept distinct from SSEConnectionError so the caller can re-check its own deadline and
|
|
30
|
+
resubscribe without consuming a connection-retry budget.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
|
|
26
34
|
class SSEClient:
|
|
27
35
|
def __init__(self, platform_url: str, token: str) -> None:
|
|
28
36
|
self.platform_url = platform_url.rstrip("/")
|
|
29
37
|
self.token = token
|
|
30
38
|
|
|
31
|
-
def stream(
|
|
39
|
+
def stream(
|
|
40
|
+
self, namespace: str, ser: str, read_timeout: float | None = None
|
|
41
|
+
) -> Iterator[SSEEvent]:
|
|
42
|
+
"""Yield parsed SSE events.
|
|
43
|
+
|
|
44
|
+
``read_timeout`` (seconds) bounds how long a single read may block with no data;
|
|
45
|
+
on expiry an :class:`SSEReadTimeout` is raised so the caller can check its own
|
|
46
|
+
deadline (and keep Ctrl-C responsive). ``None`` blocks indefinitely (legacy).
|
|
47
|
+
"""
|
|
32
48
|
url = f"{self.platform_url}/api/sse/subscribe/{namespace}/{ser}"
|
|
33
49
|
headers = {
|
|
34
50
|
"Authorization": f"Bearer {self.token}",
|
|
@@ -37,7 +53,7 @@ class SSEClient:
|
|
|
37
53
|
"Connection": "keep-alive",
|
|
38
54
|
}
|
|
39
55
|
try:
|
|
40
|
-
timeout = httpx.Timeout(connect=10.0, read=
|
|
56
|
+
timeout = httpx.Timeout(connect=10.0, read=read_timeout, write=None, pool=None)
|
|
41
57
|
with httpx.stream("GET", url, headers=headers, timeout=timeout) as r:
|
|
42
58
|
if r.status_code in (401, 403):
|
|
43
59
|
raise SSEAuthError()
|
|
@@ -51,6 +67,8 @@ class SSEClient:
|
|
|
51
67
|
event = _parse_sse_message(message)
|
|
52
68
|
if event:
|
|
53
69
|
yield event
|
|
70
|
+
except httpx.ReadTimeout as e:
|
|
71
|
+
raise SSEReadTimeout(str(e)) from e
|
|
54
72
|
except (httpx.NetworkError, httpx.RemoteProtocolError, httpx.TimeoutException) as e:
|
|
55
73
|
raise SSEConnectionError(str(e)) from e
|
|
56
74
|
|
|
@@ -270,14 +270,27 @@ class WorkloadAPI:
|
|
|
270
270
|
},
|
|
271
271
|
)
|
|
272
272
|
|
|
273
|
-
def list(
|
|
273
|
+
def list(
|
|
274
|
+
self,
|
|
275
|
+
owner: str | None = None,
|
|
276
|
+
org_code: str | None = None,
|
|
277
|
+
class_id: int | None = None,
|
|
278
|
+
) -> builtins.list[Workload]:
|
|
274
279
|
"""List workloads.
|
|
275
280
|
|
|
276
281
|
``GET /api/workloads/gai/list?owner={email}``
|
|
282
|
+
|
|
283
|
+
EDU admins/staff (``ROLE_ADMIN``, ``ROLE_EDU_STAFF``) may pass
|
|
284
|
+
``org_code`` and ``class_id`` together to retrieve every workload
|
|
285
|
+
belonging to that class.
|
|
277
286
|
"""
|
|
278
287
|
params: dict[str, str] = {}
|
|
279
288
|
if owner is not None:
|
|
280
289
|
params["owner"] = owner
|
|
290
|
+
if org_code is not None:
|
|
291
|
+
params["orgCode"] = org_code
|
|
292
|
+
if class_id is not None:
|
|
293
|
+
params["classId"] = str(class_id)
|
|
281
294
|
data = self._client.get(_GAI_BASE + "list", params=params)
|
|
282
295
|
items: builtins.list[dict[str, Any]] = data.get(
|
|
283
296
|
"workloads", data.get("data", data.get("items", []))
|
|
@@ -8,13 +8,15 @@ PBI-014 (update + camelCase unification).
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import base64
|
|
11
|
+
import contextlib
|
|
11
12
|
import json as _json
|
|
12
13
|
import math
|
|
13
14
|
import re
|
|
14
15
|
import sys
|
|
15
16
|
import time
|
|
17
|
+
from collections.abc import Callable
|
|
16
18
|
from dataclasses import dataclass, field
|
|
17
|
-
from datetime import datetime
|
|
19
|
+
from datetime import datetime, timezone
|
|
18
20
|
from typing import Any
|
|
19
21
|
|
|
20
22
|
import click
|
|
@@ -25,7 +27,7 @@ from rich.text import Text
|
|
|
25
27
|
|
|
26
28
|
from gcube_cli._group import OrderedGroup
|
|
27
29
|
from gcube_cli.api.client import APIError, AuthError, Client
|
|
28
|
-
from gcube_cli.api.sse import SSEAuthError, SSEClient, SSEConnectionError
|
|
30
|
+
from gcube_cli.api.sse import SSEAuthError, SSEClient, SSEConnectionError, SSEReadTimeout
|
|
29
31
|
from gcube_cli.api.user import UserAPI
|
|
30
32
|
from gcube_cli.api.workload import GpuPricing, Workload, WorkloadAPI, is_gpu_available
|
|
31
33
|
from gcube_cli.config.config import Config
|
|
@@ -152,6 +154,22 @@ class DeployState:
|
|
|
152
154
|
deploy_progress: list[int] = field(default_factory=list, init=False) # per-replica download %
|
|
153
155
|
# podName → replica slot, assigned in first-seen order (see _slot_for_pod)
|
|
154
156
|
pod_slots: dict[str, int] = field(default_factory=dict, init=False)
|
|
157
|
+
# Server timestamp of the event currently being processed (set in update()), so
|
|
158
|
+
# _add_log stamps log lines with the platform's time instead of the local clock.
|
|
159
|
+
_current_ts: str | None = field(default=None, init=False)
|
|
160
|
+
# Local HH:MM:SS at which each stage reached "done" (replica 0 timeline), shown
|
|
161
|
+
# next to the stage in the single-replica detail view. Stamped once per stage.
|
|
162
|
+
stage_done_time: list[str | None] = field(
|
|
163
|
+
default_factory=lambda: [None, None, None, None], init=False
|
|
164
|
+
)
|
|
165
|
+
# Server timestamps for the final result: when this deployment was created
|
|
166
|
+
# (depCreatedAt, first seen) and when it finished (timestamp of the is_done event).
|
|
167
|
+
deploy_started_ts: str | None = field(default=None, init=False)
|
|
168
|
+
done_ts: str | None = field(default=None, init=False)
|
|
169
|
+
# Optional callback invoked by _add_log with (local_time, message) — used by the
|
|
170
|
+
# non-interactive (plain) watch path to stream each event line immediately. None in
|
|
171
|
+
# the Live dashboard path, which renders from recent_logs on its own refresh tick.
|
|
172
|
+
log_sink: Callable[[str, str], None] | None = field(default=None, init=False)
|
|
155
173
|
|
|
156
174
|
def __post_init__(self) -> None:
|
|
157
175
|
self.replicas = [ReplicaStage() for _ in range(max(self.replica_count, 1))]
|
|
@@ -165,6 +183,7 @@ class DeployState:
|
|
|
165
183
|
|
|
166
184
|
def update(self, event: Any) -> None:
|
|
167
185
|
d = event.data
|
|
186
|
+
self._current_ts = d.get("timestamp")
|
|
168
187
|
event_type = d.get("eventType")
|
|
169
188
|
raw_replica = d.get("replica", 0)
|
|
170
189
|
|
|
@@ -178,6 +197,26 @@ class DeployState:
|
|
|
178
197
|
elif "replicas" in d:
|
|
179
198
|
self._handle_snapshot(d)
|
|
180
199
|
|
|
200
|
+
dep_created = d.get("depCreatedAt")
|
|
201
|
+
if dep_created and self.deploy_started_ts is None:
|
|
202
|
+
self.deploy_started_ts = dep_created
|
|
203
|
+
self._record_stage_done_times()
|
|
204
|
+
if self.is_done and self.done_ts is None:
|
|
205
|
+
self.done_ts = self._current_ts
|
|
206
|
+
|
|
207
|
+
def _record_stage_done_times(self) -> None:
|
|
208
|
+
"""Stamp the server time each stage reached "done" (replica 0 timeline).
|
|
209
|
+
|
|
210
|
+
Uses displayed_stages so stages inferred-done (vm/node skipped on tier2/3, or
|
|
211
|
+
a fast snapshot completing several at once) are stamped with the timestamp of
|
|
212
|
+
the event that advanced the deploy. Falls back to the local clock when the
|
|
213
|
+
triggering event carried no timestamp (e.g. Format B snapshots). Stamped once
|
|
214
|
+
per stage — the first time it shows done.
|
|
215
|
+
"""
|
|
216
|
+
for i, s in enumerate(self.replicas[0].displayed_stages()):
|
|
217
|
+
if s == "done" and self.stage_done_time[i] is None:
|
|
218
|
+
self.stage_done_time[i] = self._fmt_event_time(self._current_ts)
|
|
219
|
+
|
|
181
220
|
def _clamp(self, r: int) -> int:
|
|
182
221
|
return max(0, min(r, len(self.replicas) - 1))
|
|
183
222
|
|
|
@@ -373,10 +412,47 @@ class DeployState:
|
|
|
373
412
|
# No replica label: event logs come from Format A, whose `replica` is always 0,
|
|
374
413
|
# so a "[replica-N]" prefix would be misleading. Per-replica progress lives in
|
|
375
414
|
# the stage view (driven by Format B), not the log lines.
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
self.recent_logs.insert(0, entry)
|
|
415
|
+
ts = self._fmt_event_time(self._current_ts)
|
|
416
|
+
self.recent_logs.insert(0, f" {ts} {msg}")
|
|
379
417
|
self.recent_logs = self.recent_logs[:MAX_LOG_LINES]
|
|
418
|
+
if self.log_sink is not None:
|
|
419
|
+
self.log_sink(ts, msg)
|
|
420
|
+
|
|
421
|
+
def result(self, ser: str) -> dict[str, Any]:
|
|
422
|
+
"""Machine-readable terminal result of the deploy (for ``-o json``/``yaml``).
|
|
423
|
+
|
|
424
|
+
Times are the server's own timestamps (``depCreatedAt`` → the is_done event),
|
|
425
|
+
so the duration is the real elapsed time regardless of when the CLI attached.
|
|
426
|
+
"""
|
|
427
|
+
status = "failed" if self.is_failed else ("success" if self.is_done else "incomplete")
|
|
428
|
+
return {
|
|
429
|
+
"ser": _ser_value(ser),
|
|
430
|
+
"status": status,
|
|
431
|
+
"startedAt": self.deploy_started_ts,
|
|
432
|
+
"completedAt": self.done_ts,
|
|
433
|
+
"durationSec": _duration_seconds(self.deploy_started_ts, self.done_ts),
|
|
434
|
+
"failReason": (self.fail_reason or None) if self.is_failed else None,
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
@staticmethod
|
|
438
|
+
def _fmt_event_time(raw: str | None) -> str:
|
|
439
|
+
"""Format a server event timestamp as local HH:MM:SS.
|
|
440
|
+
|
|
441
|
+
The platform sends UTC timestamps in two shapes: node events use a naive
|
|
442
|
+
``"YYYY-MM-DD HH:MM:SS"`` (no offset → assumed UTC) and workload events use
|
|
443
|
+
ISO-8601 with an explicit ``+00:00`` offset. Both are converted to the local
|
|
444
|
+
timezone. Falls back to the local clock when the field is missing/unparseable
|
|
445
|
+
(e.g. Format B snapshots carry no timestamp, but they emit no log lines anyway).
|
|
446
|
+
"""
|
|
447
|
+
if raw:
|
|
448
|
+
try:
|
|
449
|
+
dt = datetime.fromisoformat(raw)
|
|
450
|
+
if dt.tzinfo is None:
|
|
451
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
452
|
+
return dt.astimezone().strftime("%H:%M:%S")
|
|
453
|
+
except ValueError:
|
|
454
|
+
pass
|
|
455
|
+
return datetime.now().strftime("%H:%M:%S")
|
|
380
456
|
|
|
381
457
|
|
|
382
458
|
# ---------------------------------------------------------------------------
|
|
@@ -384,6 +460,44 @@ class DeployState:
|
|
|
384
460
|
# ---------------------------------------------------------------------------
|
|
385
461
|
|
|
386
462
|
|
|
463
|
+
def _ser_value(ser: str) -> int | str:
|
|
464
|
+
"""SER as an int when numeric (cleaner JSON), otherwise the raw string."""
|
|
465
|
+
return int(ser) if str(ser).isdigit() else ser
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _emit_result(output: str, result: dict[str, Any]) -> None:
|
|
469
|
+
"""Print a single machine-readable deploy result object for ``-o json``/``yaml``."""
|
|
470
|
+
if output == "yaml":
|
|
471
|
+
from gcube_cli.output.yaml_out import print_yaml
|
|
472
|
+
|
|
473
|
+
print_yaml(result)
|
|
474
|
+
else:
|
|
475
|
+
from gcube_cli.output.json_out import print_json
|
|
476
|
+
|
|
477
|
+
print_json(result)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _duration_seconds(start_raw: str | None, done_raw: str | None) -> int | None:
|
|
481
|
+
"""Whole seconds between two server timestamps, or None if either is missing/unparseable.
|
|
482
|
+
|
|
483
|
+
Accepts both timestamp shapes (naive UTC ``"YYYY-MM-DD HH:MM:SS"`` and ISO-8601 with
|
|
484
|
+
offset). Clamped to >= 0 so clock skew between events never yields a negative duration.
|
|
485
|
+
"""
|
|
486
|
+
def _parse(raw: str | None) -> datetime | None:
|
|
487
|
+
if not raw:
|
|
488
|
+
return None
|
|
489
|
+
try:
|
|
490
|
+
dt = datetime.fromisoformat(raw)
|
|
491
|
+
except (ValueError, TypeError):
|
|
492
|
+
return None
|
|
493
|
+
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
|
|
494
|
+
|
|
495
|
+
start, done = _parse(start_raw), _parse(done_raw)
|
|
496
|
+
if start is None or done is None:
|
|
497
|
+
return None
|
|
498
|
+
return max(0, int((done - start).total_seconds()))
|
|
499
|
+
|
|
500
|
+
|
|
387
501
|
def _container_index(name: str) -> int:
|
|
388
502
|
"""Extract the app container index from a snapshot container name.
|
|
389
503
|
|
|
@@ -462,6 +576,8 @@ def _render(state: DeployState) -> Any:
|
|
|
462
576
|
prog = state.deploy_progress[0] if state.deploy_progress else -1
|
|
463
577
|
if prog >= 0:
|
|
464
578
|
suffix += f" {prog}%"
|
|
579
|
+
if stages[i] == "done" and state.stage_done_time[i]:
|
|
580
|
+
suffix += f" {state.stage_done_time[i]}"
|
|
465
581
|
lines.append(Text.from_markup(f" \\[{icon}] {name}{suffix}"))
|
|
466
582
|
else:
|
|
467
583
|
for r_idx, rep in enumerate(state.replicas):
|
|
@@ -969,8 +1085,22 @@ def _watch_deploy(
|
|
|
969
1085
|
client: Client,
|
|
970
1086
|
*,
|
|
971
1087
|
verb: str = "Deploying",
|
|
1088
|
+
plain: bool = False,
|
|
1089
|
+
timeout: int | None = None,
|
|
972
1090
|
) -> None:
|
|
973
|
-
"""Stream SSE deploy events and render
|
|
1091
|
+
"""Stream SSE deploy events and render deployment status until done.
|
|
1092
|
+
|
|
1093
|
+
Rendering mode is chosen automatically:
|
|
1094
|
+
- **Live dashboard** (default) on an interactive terminal.
|
|
1095
|
+
- **Plain** line-per-event stream when stdout is not a TTY (piped/redirected),
|
|
1096
|
+
when ``plain`` is forced, or under ``-o json``/``yaml`` — so batch scripts get
|
|
1097
|
+
clean, parseable output without a pseudo-terminal wrapper.
|
|
1098
|
+
With ``-o json``/``yaml`` only a single machine-readable result object is emitted.
|
|
1099
|
+
"""
|
|
1100
|
+
output = getattr(ctx.obj, "output", "table")
|
|
1101
|
+
structured = output in ("json", "yaml")
|
|
1102
|
+
# Non-interactive whenever output isn't a live terminal, is forced, or is structured.
|
|
1103
|
+
plain = structured or plain or (not sys.stdout.isatty())
|
|
974
1104
|
cfg = Config.load()
|
|
975
1105
|
api = WorkloadAPI(client)
|
|
976
1106
|
user_api = UserAPI(client)
|
|
@@ -1002,7 +1132,20 @@ def _watch_deploy(
|
|
|
1002
1132
|
# just hang with no events (e.g. watching a long-stopped workload).
|
|
1003
1133
|
phase, is_deploying = _derive_phase(wl)
|
|
1004
1134
|
if not is_deploying:
|
|
1005
|
-
if
|
|
1135
|
+
if structured:
|
|
1136
|
+
_emit_result(
|
|
1137
|
+
output,
|
|
1138
|
+
{
|
|
1139
|
+
"ser": _ser_value(ser),
|
|
1140
|
+
"status": "skipped",
|
|
1141
|
+
"phase": phase,
|
|
1142
|
+
"startedAt": None,
|
|
1143
|
+
"completedAt": None,
|
|
1144
|
+
"durationSec": None,
|
|
1145
|
+
"failReason": None,
|
|
1146
|
+
},
|
|
1147
|
+
)
|
|
1148
|
+
elif phase == "not deployed":
|
|
1006
1149
|
click.echo(f"Workload {ser} is not deployed yet. Run: gcube workload start {ser}")
|
|
1007
1150
|
else:
|
|
1008
1151
|
click.echo(f"Workload {ser} is already {phase}.")
|
|
@@ -1023,9 +1166,17 @@ def _watch_deploy(
|
|
|
1023
1166
|
# stale/empty config token and is rejected with 403.
|
|
1024
1167
|
sse = SSEClient(ctx.obj.platform_url, ctx.obj.token or "")
|
|
1025
1168
|
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1169
|
+
# Human preamble only when we're rendering for a person — suppressed under -o
|
|
1170
|
+
# json/yaml so stdout carries nothing but the final result object.
|
|
1171
|
+
if not structured:
|
|
1172
|
+
replica_label = f" ({wl.replica} replicas)" if wl.replica > 1 else ""
|
|
1173
|
+
click.echo(f'{verb} workload {ser} "{wl.description}"{replica_label}...')
|
|
1174
|
+
click.echo()
|
|
1175
|
+
# Redirected stdout is block-buffered, so a batch script tailing the file (or
|
|
1176
|
+
# hard-killing on timeout) would see nothing / lose buffered lines. Flush each
|
|
1177
|
+
# plain line as it is written so the log is live and durable.
|
|
1178
|
+
if plain:
|
|
1179
|
+
sys.stdout.flush()
|
|
1029
1180
|
|
|
1030
1181
|
state = DeployState(
|
|
1031
1182
|
replica_count=wl.replica,
|
|
@@ -1033,25 +1184,48 @@ def _watch_deploy(
|
|
|
1033
1184
|
images=[ct.get("containerImage", "") for ct in wl.container_array],
|
|
1034
1185
|
)
|
|
1035
1186
|
|
|
1187
|
+
# Plain (non-structured) mode: stream each event line to stdout as it arrives.
|
|
1188
|
+
if plain and not structured:
|
|
1189
|
+
def _emit_line(ts: str, msg: str) -> None:
|
|
1190
|
+
click.echo(f"{ts} [{ser}] {msg}")
|
|
1191
|
+
sys.stdout.flush()
|
|
1192
|
+
|
|
1193
|
+
state.log_sink = _emit_line
|
|
1194
|
+
|
|
1036
1195
|
MAX_RETRIES = 5
|
|
1037
1196
|
retry = 0
|
|
1038
1197
|
interrupted = False
|
|
1198
|
+
timed_out = False
|
|
1039
1199
|
|
|
1040
|
-
#
|
|
1041
|
-
#
|
|
1200
|
+
# When a deadline is set, bound each read so the loop can re-check the deadline (and
|
|
1201
|
+
# so Ctrl-C is delivered) instead of blocking forever on a silent stream. Poll cadence
|
|
1202
|
+
# is capped so an idle stream is noticed promptly; None keeps the legacy infinite read.
|
|
1203
|
+
deadline = (time.monotonic() + timeout) if timeout else None
|
|
1204
|
+
read_timeout = min(float(timeout), 15.0) if timeout else None
|
|
1205
|
+
|
|
1206
|
+
# --debug dumps raw SSE events to a file (the Live dashboard owns the screen, so a
|
|
1207
|
+
# file is used instead of stderr). The "→ file" notice goes to stderr to keep both
|
|
1208
|
+
# stdout streams (plain log / json result) clean.
|
|
1042
1209
|
debug = bool(getattr(ctx.obj, "debug", False))
|
|
1043
1210
|
dbg_fh = open(f"gcube-sse-{ser}.log", "w", encoding="utf-8") if debug else None
|
|
1044
1211
|
if dbg_fh:
|
|
1045
|
-
click.echo(f"[debug] raw SSE events → gcube-sse-{ser}.log")
|
|
1046
|
-
|
|
1047
|
-
#
|
|
1048
|
-
#
|
|
1049
|
-
|
|
1050
|
-
|
|
1212
|
+
click.echo(f"[debug] raw SSE events → gcube-sse-{ser}.log", err=True)
|
|
1213
|
+
|
|
1214
|
+
# Live dashboard only on an interactive terminal; plain mode runs the same loop with
|
|
1215
|
+
# no live region (events are streamed via state.log_sink instead).
|
|
1216
|
+
live_ctx: Any = (
|
|
1217
|
+
contextlib.nullcontext()
|
|
1218
|
+
if plain
|
|
1219
|
+
else Live(_LiveView(state), refresh_per_second=4, auto_refresh=True)
|
|
1220
|
+
)
|
|
1221
|
+
with live_ctx:
|
|
1051
1222
|
try:
|
|
1052
1223
|
while retry <= MAX_RETRIES:
|
|
1224
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1225
|
+
timed_out = True
|
|
1226
|
+
break
|
|
1053
1227
|
try:
|
|
1054
|
-
for event in sse.stream(namespace, ser):
|
|
1228
|
+
for event in sse.stream(namespace, ser, read_timeout=read_timeout):
|
|
1055
1229
|
if dbg_fh:
|
|
1056
1230
|
raw = _json.dumps(event.data, ensure_ascii=False)
|
|
1057
1231
|
dbg_fh.write(f"{event.event}\t{raw}\n")
|
|
@@ -1059,10 +1233,17 @@ def _watch_deploy(
|
|
|
1059
1233
|
state.update(event)
|
|
1060
1234
|
if state.is_done:
|
|
1061
1235
|
break
|
|
1236
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
1237
|
+
timed_out = True
|
|
1238
|
+
break
|
|
1062
1239
|
else:
|
|
1063
1240
|
# Generator exhausted without completion → server closed stream
|
|
1064
1241
|
raise SSEConnectionError("Stream closed by server")
|
|
1065
|
-
break # is_done
|
|
1242
|
+
break # is_done or timed_out — exit retry loop
|
|
1243
|
+
except SSEReadTimeout:
|
|
1244
|
+
# Idle gap (no data within read_timeout). Not a failure: loop back to
|
|
1245
|
+
# re-check the deadline and resubscribe without spending a retry.
|
|
1246
|
+
continue
|
|
1066
1247
|
except SSEAuthError:
|
|
1067
1248
|
click.echo(
|
|
1068
1249
|
"Token expired or invalid. "
|
|
@@ -1083,11 +1264,40 @@ def _watch_deploy(
|
|
|
1083
1264
|
dbg_fh.close()
|
|
1084
1265
|
|
|
1085
1266
|
if interrupted:
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1267
|
+
if structured:
|
|
1268
|
+
res = state.result(ser)
|
|
1269
|
+
res["status"] = "interrupted"
|
|
1270
|
+
_emit_result(output, res)
|
|
1271
|
+
else:
|
|
1272
|
+
click.echo()
|
|
1273
|
+
click.echo(
|
|
1274
|
+
f"Monitoring stopped. Workload {ser} continues deploying in the background."
|
|
1275
|
+
)
|
|
1276
|
+
_hint(f"Re-attach: gcube workload watch {ser}")
|
|
1277
|
+
return
|
|
1278
|
+
|
|
1279
|
+
if timed_out:
|
|
1280
|
+
# Inconclusive: deploy neither completed nor reported terminal failure within the
|
|
1281
|
+
# deadline. Exit 124 (GNU `timeout` convention) so batch scripts can branch on it.
|
|
1282
|
+
if structured:
|
|
1283
|
+
res = state.result(ser)
|
|
1284
|
+
res["status"] = "timeout"
|
|
1285
|
+
_emit_result(output, res)
|
|
1286
|
+
else:
|
|
1287
|
+
click.echo()
|
|
1288
|
+
click.echo(
|
|
1289
|
+
f"Watch timed out after {timeout}s — workload {ser} did not finish "
|
|
1290
|
+
"deploying. It continues in the background.",
|
|
1291
|
+
err=True,
|
|
1292
|
+
)
|
|
1293
|
+
_hint(f"Re-attach: gcube workload watch {ser}")
|
|
1294
|
+
sys.exit(124)
|
|
1295
|
+
|
|
1296
|
+
# Structured: emit only the result object (no human text on stdout).
|
|
1297
|
+
if structured:
|
|
1298
|
+
_emit_result(output, state.result(ser))
|
|
1299
|
+
if state.is_failed:
|
|
1300
|
+
sys.exit(1)
|
|
1091
1301
|
return
|
|
1092
1302
|
|
|
1093
1303
|
click.echo()
|
|
@@ -1104,7 +1314,9 @@ def _watch_deploy(
|
|
|
1104
1314
|
err=True,
|
|
1105
1315
|
)
|
|
1106
1316
|
sys.exit(1)
|
|
1107
|
-
|
|
1317
|
+
dur = _duration_seconds(state.deploy_started_ts, state.done_ts)
|
|
1318
|
+
took = f" (took {dur}s)" if dur is not None else ""
|
|
1319
|
+
click.echo(f"Workload {ser} deployed successfully.{took}")
|
|
1108
1320
|
_hint(f"View logs: gcube workload logs {ser}")
|
|
1109
1321
|
_hint(f"Details: gcube workload describe {ser}")
|
|
1110
1322
|
|
|
@@ -1448,13 +1660,34 @@ def workload_update(
|
|
|
1448
1660
|
|
|
1449
1661
|
@workload.command("list")
|
|
1450
1662
|
@click.option("--owner", default=None, help="Filter by owner email")
|
|
1663
|
+
@click.option(
|
|
1664
|
+
"--org-code",
|
|
1665
|
+
default=None,
|
|
1666
|
+
help="EDU 단체코드 (EDU 관리자/운영자 전용, --class-id와 함께 사용)",
|
|
1667
|
+
)
|
|
1668
|
+
@click.option(
|
|
1669
|
+
"--class-id",
|
|
1670
|
+
type=int,
|
|
1671
|
+
default=None,
|
|
1672
|
+
help="EDU 클래스 ID (= 그룹 ID, 그룹 목록 조회에서 확인, --org-code와 함께 사용)",
|
|
1673
|
+
)
|
|
1451
1674
|
@click.pass_context
|
|
1452
|
-
def workload_list(
|
|
1675
|
+
def workload_list(
|
|
1676
|
+
ctx: click.Context,
|
|
1677
|
+
owner: str | None,
|
|
1678
|
+
org_code: str | None,
|
|
1679
|
+
class_id: int | None,
|
|
1680
|
+
) -> None:
|
|
1453
1681
|
"""List workloads."""
|
|
1682
|
+
if (org_code is None) != (class_id is None):
|
|
1683
|
+
click.echo(
|
|
1684
|
+
"--org-code 와 --class-id 는 함께 사용해야 합니다.", err=True
|
|
1685
|
+
)
|
|
1686
|
+
sys.exit(1)
|
|
1454
1687
|
try:
|
|
1455
1688
|
client = _make_client(ctx)
|
|
1456
1689
|
api = WorkloadAPI(client)
|
|
1457
|
-
workloads = api.list(owner=owner)
|
|
1690
|
+
workloads = api.list(owner=owner, org_code=org_code, class_id=class_id)
|
|
1458
1691
|
|
|
1459
1692
|
output = ctx.obj.output
|
|
1460
1693
|
list_dicts = [
|
|
@@ -1533,8 +1766,23 @@ def workload_describe(ctx: click.Context, ser: str) -> None:
|
|
|
1533
1766
|
default=False,
|
|
1534
1767
|
help="Start workload without monitoring deployment status",
|
|
1535
1768
|
)
|
|
1769
|
+
@click.option(
|
|
1770
|
+
"--plain",
|
|
1771
|
+
is_flag=True,
|
|
1772
|
+
default=False,
|
|
1773
|
+
help="Stream plain line-per-event output instead of the live dashboard "
|
|
1774
|
+
"(auto-enabled when output is piped/redirected or with -o json/yaml)",
|
|
1775
|
+
)
|
|
1776
|
+
@click.option(
|
|
1777
|
+
"--timeout",
|
|
1778
|
+
type=int,
|
|
1779
|
+
default=None,
|
|
1780
|
+
help="Stop watching after N seconds if the deploy hasn't finished (exit code 124)",
|
|
1781
|
+
)
|
|
1536
1782
|
@click.pass_context
|
|
1537
|
-
def workload_start(
|
|
1783
|
+
def workload_start(
|
|
1784
|
+
ctx: click.Context, ser: str, no_watch: bool, plain: bool, timeout: int | None
|
|
1785
|
+
) -> None:
|
|
1538
1786
|
"""Deploy (start) a workload (and monitor deployment by default)."""
|
|
1539
1787
|
client = _make_client(ctx)
|
|
1540
1788
|
try:
|
|
@@ -1551,7 +1799,7 @@ def workload_start(ctx: click.Context, ser: str, no_watch: bool) -> None:
|
|
|
1551
1799
|
click.echo(f"Workload {ser} started.")
|
|
1552
1800
|
return
|
|
1553
1801
|
|
|
1554
|
-
_watch_deploy(ctx, ser, client)
|
|
1802
|
+
_watch_deploy(ctx, ser, client, plain=plain, timeout=timeout)
|
|
1555
1803
|
|
|
1556
1804
|
|
|
1557
1805
|
# ---------------------------------------------------------------------------
|
|
@@ -1561,11 +1809,24 @@ def workload_start(ctx: click.Context, ser: str, no_watch: bool) -> None:
|
|
|
1561
1809
|
|
|
1562
1810
|
@workload.command("watch")
|
|
1563
1811
|
@click.argument("ser")
|
|
1812
|
+
@click.option(
|
|
1813
|
+
"--plain",
|
|
1814
|
+
is_flag=True,
|
|
1815
|
+
default=False,
|
|
1816
|
+
help="Stream plain line-per-event output instead of the live dashboard "
|
|
1817
|
+
"(auto-enabled when output is piped/redirected or with -o json/yaml)",
|
|
1818
|
+
)
|
|
1819
|
+
@click.option(
|
|
1820
|
+
"--timeout",
|
|
1821
|
+
type=int,
|
|
1822
|
+
default=None,
|
|
1823
|
+
help="Stop watching after N seconds if the deploy hasn't finished (exit code 124)",
|
|
1824
|
+
)
|
|
1564
1825
|
@click.pass_context
|
|
1565
|
-
def workload_watch(ctx: click.Context, ser: str) -> None:
|
|
1826
|
+
def workload_watch(ctx: click.Context, ser: str, plain: bool, timeout: int | None) -> None:
|
|
1566
1827
|
"""Monitor real-time deployment status of a running workload."""
|
|
1567
1828
|
client = _make_client(ctx)
|
|
1568
|
-
_watch_deploy(ctx, ser, client, verb="Connecting to")
|
|
1829
|
+
_watch_deploy(ctx, ser, client, verb="Connecting to", plain=plain, timeout=timeout)
|
|
1569
1830
|
|
|
1570
1831
|
|
|
1571
1832
|
# ---------------------------------------------------------------------------
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gcube-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Official command-line tool for the gcube AI GPU cloud platform
|
|
5
5
|
Author-email: Data Alliance <support@data-alliance.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -162,7 +162,9 @@ Manage the full lifecycle of GPU workloads. Each workload is identified by a **S
|
|
|
162
162
|
| `gcube workload describe <ser>` | Show workload details |
|
|
163
163
|
| `gcube workload start <ser>` | Start a workload and monitor deployment status in real time |
|
|
164
164
|
| `gcube workload start <ser> --no-watch` | Start a workload without monitoring |
|
|
165
|
+
| `gcube workload start <ser> --timeout <sec>` | Stop monitoring after N seconds if the deploy hasn't finished (exit code 124) |
|
|
165
166
|
| `gcube workload watch <ser>` | Re-attach to a deploying workload and monitor its status |
|
|
167
|
+
| `gcube workload watch <ser> --plain` | Force plain line-per-event output instead of the live dashboard |
|
|
166
168
|
| `gcube workload stop <ser>` | Stop a workload |
|
|
167
169
|
| `gcube workload delete <ser>` | Delete a workload |
|
|
168
170
|
| `gcube workload logs <ser>` | Stream container logs in real time |
|
|
@@ -248,6 +250,14 @@ gpuSpecs:
|
|
|
248
250
|
- Use `--no-watch` to start without monitoring (e.g. in scripts).
|
|
249
251
|
- For deployments with more than 5 replicas, progress is shown as an aggregate counter (e.g. `Deploying... 60/100`) instead of per-stage detail.
|
|
250
252
|
|
|
253
|
+
**Non-interactive / scripting**
|
|
254
|
+
|
|
255
|
+
`start` and `watch` adapt their output for automation:
|
|
256
|
+
|
|
257
|
+
- When stdout is **not a terminal** (piped or redirected), or with `--plain`, output switches from the live dashboard to a plain line-per-event stream (`HH:MM:SS [ser] message`, server timestamps, flushed per line — safe to `tail -f` and durable even if the process is hard-killed).
|
|
258
|
+
- With `-o json` / `-o yaml`, only a single result object is printed: `{ser, status, startedAt, completedAt, durationSec, failReason}`. `status` is one of `success`, `failed`, `timeout`, `interrupted`, `skipped`.
|
|
259
|
+
- `--timeout <sec>` stops an unfinished watch after N seconds and exits `124` (GNU `timeout` convention); it also keeps `Ctrl+C` responsive on a silent stream.
|
|
260
|
+
|
|
251
261
|
---
|
|
252
262
|
|
|
253
263
|
### GPU
|
|
@@ -355,6 +365,7 @@ gcube -o yaml workload list # YAML
|
|
|
355
365
|
| `2` | gcube API error |
|
|
356
366
|
| `3` | Authentication failure or token expired |
|
|
357
367
|
| `4` | Network error |
|
|
368
|
+
| `124` | `workload watch`/`start` timed out (`--timeout`) before the deploy finished |
|
|
358
369
|
|
|
359
370
|
---
|
|
360
371
|
|
|
@@ -394,6 +405,26 @@ gcube workload stop 2212
|
|
|
394
405
|
gcube workload delete 2212
|
|
395
406
|
```
|
|
396
407
|
|
|
408
|
+
### Scripting / Batch Deployment
|
|
409
|
+
|
|
410
|
+
```bash
|
|
411
|
+
# Plain log to a file (auto-detected; no live dashboard). Capture stderr too so
|
|
412
|
+
# failure/timeout messages land in the log. Branch on the exit code.
|
|
413
|
+
gcube workload watch 2212 --timeout 1800 > 2212.log 2>&1
|
|
414
|
+
echo "exit=$?" # 0 success · 1 failed · 124 timeout
|
|
415
|
+
|
|
416
|
+
# Machine-readable result for aggregating many deploys
|
|
417
|
+
gcube -o json workload watch 2212 --timeout 1800 > 2212.json
|
|
418
|
+
# {"ser":2212,"status":"success","startedAt":...,"completedAt":...,"durationSec":...,"failReason":null}
|
|
419
|
+
|
|
420
|
+
# Per-student burst with distinct tokens (one process each)
|
|
421
|
+
for ser in "${SERS[@]}"; do
|
|
422
|
+
GCUBE_ACCESS_TOKEN="${TOKENS[$ser]}" \
|
|
423
|
+
gcube -o json workload watch "$ser" --timeout 1800 > "$ser.json" &
|
|
424
|
+
done
|
|
425
|
+
wait
|
|
426
|
+
```
|
|
427
|
+
|
|
397
428
|
### Update a Workload
|
|
398
429
|
|
|
399
430
|
```bash
|
|
@@ -317,6 +317,14 @@ class TestWorkloadAPIList:
|
|
|
317
317
|
request = httpx_mock.get_request()
|
|
318
318
|
assert "owner=" in str(request.url)
|
|
319
319
|
|
|
320
|
+
def test_list_with_edu_org_and_class(self, httpx_mock: Any) -> None:
|
|
321
|
+
httpx_mock.add_response(json=ok_envelope(workloads=[WORKLOAD_ITEM]))
|
|
322
|
+
make_api().list(org_code="samsung", class_id=5)
|
|
323
|
+
|
|
324
|
+
url = str(httpx_mock.get_request().url)
|
|
325
|
+
assert "orgCode=samsung" in url
|
|
326
|
+
assert "classId=5" in url
|
|
327
|
+
|
|
320
328
|
def test_list_returns_empty_list(self, httpx_mock: Any) -> None:
|
|
321
329
|
httpx_mock.add_response(json=ok_envelope(workloads=[]))
|
|
322
330
|
result = make_api().list()
|
|
@@ -203,6 +203,33 @@ class TestWorkloadList:
|
|
|
203
203
|
assert result.exit_code == 0
|
|
204
204
|
assert "owner=" in str(httpx_mock.get_request().url)
|
|
205
205
|
|
|
206
|
+
def test_edu_org_and_class_send_query_params(
|
|
207
|
+
self, runner: CliRunner, httpx_mock: Any, config_dir: Path
|
|
208
|
+
) -> None:
|
|
209
|
+
httpx_mock.add_response(json=ok_envelope(workloads=[WORKLOAD_ITEM]))
|
|
210
|
+
result = run(
|
|
211
|
+
runner,
|
|
212
|
+
["workload", "list", "--org-code", "samsung", "--class-id", "5"],
|
|
213
|
+
)
|
|
214
|
+
assert result.exit_code == 0
|
|
215
|
+
url = str(httpx_mock.get_request().url)
|
|
216
|
+
assert "orgCode=samsung" in url
|
|
217
|
+
assert "classId=5" in url
|
|
218
|
+
|
|
219
|
+
def test_org_code_without_class_id_exits_1(
|
|
220
|
+
self, runner: CliRunner, config_dir: Path
|
|
221
|
+
) -> None:
|
|
222
|
+
result = run(runner, ["workload", "list", "--org-code", "samsung"])
|
|
223
|
+
assert result.exit_code == 1
|
|
224
|
+
assert "함께 사용" in result.output
|
|
225
|
+
|
|
226
|
+
def test_class_id_without_org_code_exits_1(
|
|
227
|
+
self, runner: CliRunner, config_dir: Path
|
|
228
|
+
) -> None:
|
|
229
|
+
result = run(runner, ["workload", "list", "--class-id", "5"])
|
|
230
|
+
assert result.exit_code == 1
|
|
231
|
+
assert "함께 사용" in result.output
|
|
232
|
+
|
|
206
233
|
def test_empty_list_exits_0(
|
|
207
234
|
self, runner: CliRunner, httpx_mock: Any, config_dir: Path
|
|
208
235
|
) -> None:
|
|
@@ -15,6 +15,7 @@ import json
|
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
from typing import Any
|
|
17
17
|
|
|
18
|
+
import httpx
|
|
18
19
|
import pytest
|
|
19
20
|
from click.testing import CliRunner
|
|
20
21
|
|
|
@@ -23,6 +24,7 @@ from gcube_cli.api.sse import (
|
|
|
23
24
|
SSEClient,
|
|
24
25
|
SSEConnectionError,
|
|
25
26
|
SSEEvent,
|
|
27
|
+
SSEReadTimeout,
|
|
26
28
|
_parse_sse_message,
|
|
27
29
|
)
|
|
28
30
|
from gcube_cli.api.workload import Workload
|
|
@@ -219,6 +221,16 @@ class TestSSEClientStream:
|
|
|
219
221
|
events = list(SSEClient(PLATFORM_URL, TOKEN).stream("ns123", "2143"))
|
|
220
222
|
assert len(events) == 1
|
|
221
223
|
|
|
224
|
+
def test_read_timeout_raises_sse_read_timeout(self, httpx_mock: Any) -> None:
|
|
225
|
+
"""An idle read (httpx.ReadTimeout) maps to SSEReadTimeout, not a connection error."""
|
|
226
|
+
httpx_mock.add_exception(
|
|
227
|
+
httpx.ReadTimeout("timed out"),
|
|
228
|
+
method="GET",
|
|
229
|
+
url=f"{PLATFORM_URL}/api/sse/subscribe/ns123/2143",
|
|
230
|
+
)
|
|
231
|
+
with pytest.raises(SSEReadTimeout):
|
|
232
|
+
list(SSEClient(PLATFORM_URL, TOKEN).stream("ns123", "2143", read_timeout=1))
|
|
233
|
+
|
|
222
234
|
|
|
223
235
|
# ---------------------------------------------------------------------------
|
|
224
236
|
# DeployState — Format A events
|
|
@@ -648,6 +660,44 @@ class TestDeployStateSnapshotInProgress:
|
|
|
648
660
|
assert state.is_done # slot 0 completed (1 replica)
|
|
649
661
|
|
|
650
662
|
|
|
663
|
+
# ---------------------------------------------------------------------------
|
|
664
|
+
# Event log timestamps — use the server's UTC time (→ local), not the local clock
|
|
665
|
+
# ---------------------------------------------------------------------------
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
class TestEventLogTimestamp:
|
|
669
|
+
def _ev(self, data: dict[str, Any]) -> SSEEvent:
|
|
670
|
+
return SSEEvent(event="deployEvent", data=data)
|
|
671
|
+
|
|
672
|
+
def test_naive_timestamp_treated_as_utc(self) -> None:
|
|
673
|
+
# node format (no offset) and the same instant with explicit +00:00 must agree,
|
|
674
|
+
# regardless of the machine's local timezone.
|
|
675
|
+
naive = DeployState._fmt_event_time("2026-06-02 05:10:02")
|
|
676
|
+
aware = DeployState._fmt_event_time("2026-06-02T05:10:02+00:00")
|
|
677
|
+
assert naive == aware
|
|
678
|
+
|
|
679
|
+
def test_workload_iso_timestamp_parsed(self) -> None:
|
|
680
|
+
out = DeployState._fmt_event_time("2026-06-02T05:10:19.941647+00:00")
|
|
681
|
+
assert len(out) == 8 and out.count(":") == 2
|
|
682
|
+
|
|
683
|
+
def test_missing_timestamp_falls_back_to_local_clock(self) -> None:
|
|
684
|
+
out = DeployState._fmt_event_time(None)
|
|
685
|
+
assert len(out) == 8 and out.count(":") == 2
|
|
686
|
+
|
|
687
|
+
def test_malformed_timestamp_falls_back(self) -> None:
|
|
688
|
+
out = DeployState._fmt_event_time("not-a-timestamp")
|
|
689
|
+
assert len(out) == 8 and out.count(":") == 2
|
|
690
|
+
|
|
691
|
+
def test_log_line_uses_server_timestamp(self) -> None:
|
|
692
|
+
state = DeployState(replica_count=1, container_count=1)
|
|
693
|
+
state.update(self._ev({
|
|
694
|
+
"eventType": "workload", "reason": "image_pull_started",
|
|
695
|
+
"replica": "0", "timestamp": "2026-06-02T05:10:19+00:00",
|
|
696
|
+
}))
|
|
697
|
+
expected = DeployState._fmt_event_time("2026-06-02T05:10:19+00:00")
|
|
698
|
+
assert state.recent_logs[0].strip().startswith(expected)
|
|
699
|
+
|
|
700
|
+
|
|
651
701
|
# ---------------------------------------------------------------------------
|
|
652
702
|
# Rendering — large-scale (>5) simple completed-counter view (no per-stage bars)
|
|
653
703
|
# ---------------------------------------------------------------------------
|
|
@@ -705,6 +755,39 @@ class TestAggregateCounterRender:
|
|
|
705
755
|
assert "replica-0" in out
|
|
706
756
|
|
|
707
757
|
|
|
758
|
+
class TestSingleReplicaStageTime:
|
|
759
|
+
"""Single-replica detail view shows each stage's server completion time."""
|
|
760
|
+
|
|
761
|
+
def _ev(self, data: dict[str, Any]) -> SSEEvent:
|
|
762
|
+
return SSEEvent(event="deployEvent", data=data)
|
|
763
|
+
|
|
764
|
+
def test_done_stage_shows_server_completion_time(self) -> None:
|
|
765
|
+
state = DeployState(replica_count=1, container_count=1)
|
|
766
|
+
state.update(self._ev({
|
|
767
|
+
"eventType": "workload", "reason": "container_started",
|
|
768
|
+
"replica": "0", "containerIndex": 0,
|
|
769
|
+
"timestamp": "2026-06-02T05:10:30+00:00",
|
|
770
|
+
}))
|
|
771
|
+
expected = DeployState._fmt_event_time("2026-06-02T05:10:30+00:00")
|
|
772
|
+
assert state.is_done
|
|
773
|
+
assert all(t == expected for t in state.stage_done_time)
|
|
774
|
+
assert expected in _render_text(state)
|
|
775
|
+
|
|
776
|
+
def test_running_and_pending_stages_have_no_time(self) -> None:
|
|
777
|
+
state = DeployState(replica_count=1, container_count=1)
|
|
778
|
+
state.update(self._ev({
|
|
779
|
+
"eventType": "workload", "reason": "image_pulling",
|
|
780
|
+
"replica": "0", "downloadProgress": "50.0",
|
|
781
|
+
"timestamp": "2026-06-02T05:10:10+00:00",
|
|
782
|
+
}))
|
|
783
|
+
# vm/node are inferred-done (lower than the running stage) → stamped;
|
|
784
|
+
# the running image-pull stage and the pending container stage are not.
|
|
785
|
+
assert state.stage_done_time[STAGE_VM] is not None
|
|
786
|
+
assert state.stage_done_time[STAGE_NODE] is not None
|
|
787
|
+
assert state.stage_done_time[STAGE_DEPLOY_START] is None
|
|
788
|
+
assert state.stage_done_time[STAGE_DEPLOY_DONE] is None
|
|
789
|
+
|
|
790
|
+
|
|
708
791
|
# ---------------------------------------------------------------------------
|
|
709
792
|
# _derive_phase — pre-watch state check (operations history, not the DB-echo state)
|
|
710
793
|
# ---------------------------------------------------------------------------
|
|
@@ -918,7 +1001,15 @@ class TestWorkloadWatchCommand:
|
|
|
918
1001
|
"""watch command invokes _watch_deploy (mocked to avoid Live/SSE)."""
|
|
919
1002
|
called: dict[str, Any] = {}
|
|
920
1003
|
|
|
921
|
-
def mock_watch(
|
|
1004
|
+
def mock_watch(
|
|
1005
|
+
ctx: Any,
|
|
1006
|
+
ser: str,
|
|
1007
|
+
client: Any,
|
|
1008
|
+
*,
|
|
1009
|
+
verb: str = "Deploying",
|
|
1010
|
+
plain: bool = False,
|
|
1011
|
+
timeout: int | None = None,
|
|
1012
|
+
) -> None:
|
|
922
1013
|
called["ser"] = ser
|
|
923
1014
|
called["verb"] = verb
|
|
924
1015
|
|
|
@@ -943,7 +1034,15 @@ class TestWorkloadWatchCommand:
|
|
|
943
1034
|
)
|
|
944
1035
|
called: dict[str, Any] = {}
|
|
945
1036
|
|
|
946
|
-
def mock_watch(
|
|
1037
|
+
def mock_watch(
|
|
1038
|
+
ctx: Any,
|
|
1039
|
+
ser: str,
|
|
1040
|
+
client: Any,
|
|
1041
|
+
*,
|
|
1042
|
+
verb: str = "Deploying",
|
|
1043
|
+
plain: bool = False,
|
|
1044
|
+
timeout: int | None = None,
|
|
1045
|
+
) -> None:
|
|
947
1046
|
called["ser"] = ser
|
|
948
1047
|
called["verb"] = verb
|
|
949
1048
|
|
|
@@ -975,6 +1074,118 @@ class MockLive:
|
|
|
975
1074
|
pass
|
|
976
1075
|
|
|
977
1076
|
|
|
1077
|
+
class TestDeployResult:
|
|
1078
|
+
"""Server-timestamp based final result (for -o json) and duration helper."""
|
|
1079
|
+
|
|
1080
|
+
def _ev(self, data: dict[str, Any]) -> SSEEvent:
|
|
1081
|
+
return SSEEvent(event="deployEvent", data=data)
|
|
1082
|
+
|
|
1083
|
+
def test_duration_seconds(self) -> None:
|
|
1084
|
+
from gcube_cli.commands.workload import _duration_seconds
|
|
1085
|
+
|
|
1086
|
+
# naive(UTC) start → ISO+offset done = 9s
|
|
1087
|
+
assert _duration_seconds("2026-06-12 07:24:48", "2026-06-12T07:24:57+00:00") == 9
|
|
1088
|
+
# missing/unparseable → None
|
|
1089
|
+
assert _duration_seconds(None, "2026-06-12T07:24:57+00:00") is None
|
|
1090
|
+
assert _duration_seconds("nope", "also-nope") is None
|
|
1091
|
+
# done before start (clock skew) → clamped to 0, never negative
|
|
1092
|
+
assert _duration_seconds("2026-06-12T07:24:57+00:00", "2026-06-12T07:24:48+00:00") == 0
|
|
1093
|
+
|
|
1094
|
+
def test_result_success_uses_server_times(self) -> None:
|
|
1095
|
+
state = DeployState(replica_count=1, container_count=1)
|
|
1096
|
+
state.update(self._ev({
|
|
1097
|
+
"eventType": "workload", "reason": "container_started", "replica": "0",
|
|
1098
|
+
"containerIndex": 0, "timestamp": "2026-06-12T07:24:57+00:00",
|
|
1099
|
+
"depCreatedAt": "2026-06-12 07:24:48",
|
|
1100
|
+
}))
|
|
1101
|
+
assert state.result("2143") == {
|
|
1102
|
+
"ser": 2143,
|
|
1103
|
+
"status": "success",
|
|
1104
|
+
"startedAt": "2026-06-12 07:24:48",
|
|
1105
|
+
"completedAt": "2026-06-12T07:24:57+00:00",
|
|
1106
|
+
"durationSec": 9,
|
|
1107
|
+
"failReason": None,
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
class TestWatchPlainAndJson:
|
|
1112
|
+
"""Non-interactive paths used by batch scripts (no pseudo-terminal needed)."""
|
|
1113
|
+
|
|
1114
|
+
def _setup(self, httpx_mock: Any) -> None:
|
|
1115
|
+
httpx_mock.add_response(
|
|
1116
|
+
method="GET",
|
|
1117
|
+
url=f"{PLATFORM_URL}/api/workloads/gai/2143",
|
|
1118
|
+
json=ok_envelope(workload=WORKLOAD_ITEM),
|
|
1119
|
+
)
|
|
1120
|
+
httpx_mock.add_response(
|
|
1121
|
+
method="GET",
|
|
1122
|
+
url=f"{PLATFORM_URL}/api/users/user@example.com",
|
|
1123
|
+
json=USER_RESPONSE,
|
|
1124
|
+
)
|
|
1125
|
+
|
|
1126
|
+
_DONE_EVENT = {
|
|
1127
|
+
"eventType": "workload", "reason": "container_started", "replica": "0",
|
|
1128
|
+
"containerIndex": 0, "timestamp": "2026-06-12T07:24:57+00:00",
|
|
1129
|
+
"depCreatedAt": "2026-06-12 07:24:48",
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
def test_plain_streams_ser_tagged_lines(
|
|
1133
|
+
self, runner: CliRunner, httpx_mock: Any, config_with_email: Path
|
|
1134
|
+
) -> None:
|
|
1135
|
+
self._setup(httpx_mock)
|
|
1136
|
+
httpx_mock.add_response(
|
|
1137
|
+
method="GET",
|
|
1138
|
+
url=f"{PLATFORM_URL}/api/sse/subscribe/ns904811bf/2143",
|
|
1139
|
+
content=sse_message("deployEvent", self._DONE_EVENT).encode(),
|
|
1140
|
+
)
|
|
1141
|
+
# CliRunner stdout is not a TTY → plain mode auto-engages (no Live needed).
|
|
1142
|
+
result = run(runner, ["workload", "watch", "2143"])
|
|
1143
|
+
assert result.exit_code == 0
|
|
1144
|
+
assert "[2143]" in result.output # ser-tagged event line
|
|
1145
|
+
assert "deployed successfully" in result.output
|
|
1146
|
+
|
|
1147
|
+
def test_json_emits_single_result_object(
|
|
1148
|
+
self, runner: CliRunner, httpx_mock: Any, config_with_email: Path
|
|
1149
|
+
) -> None:
|
|
1150
|
+
self._setup(httpx_mock)
|
|
1151
|
+
httpx_mock.add_response(
|
|
1152
|
+
method="GET",
|
|
1153
|
+
url=f"{PLATFORM_URL}/api/sse/subscribe/ns904811bf/2143",
|
|
1154
|
+
content=sse_message("deployEvent", self._DONE_EVENT).encode(),
|
|
1155
|
+
)
|
|
1156
|
+
result = run(runner, ["--output", "json", "workload", "watch", "2143"])
|
|
1157
|
+
assert result.exit_code == 0
|
|
1158
|
+
data = json.loads(result.output)
|
|
1159
|
+
assert data["ser"] == 2143
|
|
1160
|
+
assert data["status"] == "success"
|
|
1161
|
+
assert data["durationSec"] == 9
|
|
1162
|
+
assert data["startedAt"] and data["completedAt"]
|
|
1163
|
+
|
|
1164
|
+
def test_timeout_exits_124_with_status_timeout(
|
|
1165
|
+
self,
|
|
1166
|
+
runner: CliRunner,
|
|
1167
|
+
httpx_mock: Any,
|
|
1168
|
+
config_with_email: Path,
|
|
1169
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
1170
|
+
) -> None:
|
|
1171
|
+
"""--timeout: if the deadline passes before completion, exit 124, status=timeout.
|
|
1172
|
+
|
|
1173
|
+
The clock is advanced past the deadline so the loop times out on its first check
|
|
1174
|
+
(before any SSE subscribe), keeping the test deterministic and fast."""
|
|
1175
|
+
self._setup(httpx_mock)
|
|
1176
|
+
# monotonic(): 1st call computes the deadline (=0+5), every later call is past it.
|
|
1177
|
+
clock = iter([0.0])
|
|
1178
|
+
monkeypatch.setattr(
|
|
1179
|
+
"gcube_cli.commands.workload.time.monotonic",
|
|
1180
|
+
lambda: next(clock, 1000.0),
|
|
1181
|
+
)
|
|
1182
|
+
result = run(runner, ["--output", "json", "workload", "watch", "2143", "--timeout", "5"])
|
|
1183
|
+
assert result.exit_code == 124
|
|
1184
|
+
data = json.loads(result.output)
|
|
1185
|
+
assert data["ser"] == 2143
|
|
1186
|
+
assert data["status"] == "timeout"
|
|
1187
|
+
|
|
1188
|
+
|
|
978
1189
|
class TestWatchDeployIntegration:
|
|
979
1190
|
def test_watch_skips_sse_for_stopped_workload(
|
|
980
1191
|
self,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|