kdev-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kdev/__init__.py +7 -0
- kdev/__main__.py +3 -0
- kdev/api.py +408 -0
- kdev/auth.py +115 -0
- kdev/bootstrap.py +510 -0
- kdev/cli.py +162 -0
- kdev/cloudflared.py +179 -0
- kdev/commands/__init__.py +5 -0
- kdev/commands/_common.py +51 -0
- kdev/commands/account.py +148 -0
- kdev/commands/box.py +694 -0
- kdev/commands/doctor.py +127 -0
- kdev/commands/home.py +92 -0
- kdev/commands/settings.py +164 -0
- kdev/commands/tunnel.py +78 -0
- kdev/commands/workspace.py +210 -0
- kdev/config.py +181 -0
- kdev/errors.py +25 -0
- kdev/kdev_box.py +538 -0
- kdev/notebook.py +261 -0
- kdev/persistence.py +314 -0
- kdev/quota.py +102 -0
- kdev/safety.py +72 -0
- kdev/session.py +284 -0
- kdev/sshcfg.py +72 -0
- kdev/tunnel.py +259 -0
- kdev/ui.py +666 -0
- kdev/wizard.py +251 -0
- kdev_cli-0.1.3.dist-info/METADATA +598 -0
- kdev_cli-0.1.3.dist-info/RECORD +33 -0
- kdev_cli-0.1.3.dist-info/WHEEL +4 -0
- kdev_cli-0.1.3.dist-info/entry_points.txt +2 -0
- kdev_cli-0.1.3.dist-info/licenses/LICENSE +21 -0
kdev/commands/box.py
ADDED
|
@@ -0,0 +1,694 @@
|
|
|
1
|
+
"""The box: start it, reach it, see it, stop it, and get your files back."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import subprocess
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from rich.text import Text
|
|
11
|
+
|
|
12
|
+
from .. import (
|
|
13
|
+
api,
|
|
14
|
+
bootstrap,
|
|
15
|
+
cloudflared,
|
|
16
|
+
config,
|
|
17
|
+
persistence,
|
|
18
|
+
quota,
|
|
19
|
+
safety,
|
|
20
|
+
session,
|
|
21
|
+
sshcfg,
|
|
22
|
+
ui,
|
|
23
|
+
)
|
|
24
|
+
from .. import notebook as nb
|
|
25
|
+
from ..errors import KdevError
|
|
26
|
+
from ._common import cap_for, context, load_ready, need_notebook
|
|
27
|
+
|
|
28
|
+
OPEN_CHOICES = ("code", "shell", "none")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# --- up -----------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def up(
|
|
35
|
+
gpu: str | None = typer.Option(None, "--gpu", help=f"Accelerator: {', '.join(api.SHAPES)}."),
|
|
36
|
+
hours: float | None = typer.Option(
|
|
37
|
+
None, "--hours", help="Session length (Kaggle stops it at 12h, 9h on TPU)."
|
|
38
|
+
),
|
|
39
|
+
account: str = typer.Option("", "--account", "-a", help="Run it on this account."),
|
|
40
|
+
open_with: str | None = typer.Option(None, "--open", help="When ready: code, shell or none."),
|
|
41
|
+
restore: bool = typer.Option(True, "--restore/--no-restore", help="Put your saved files back."),
|
|
42
|
+
replace: bool = typer.Option(False, "--replace", help="Replace a box that is already running."),
|
|
43
|
+
yes: bool = typer.Option(False, "--yes", "-y", help="No questions: use flags and defaults."),
|
|
44
|
+
) -> None:
|
|
45
|
+
"""Start the box, or connect to the one already running."""
|
|
46
|
+
interactive = ui.interactive() and not yes
|
|
47
|
+
if open_with is not None and open_with not in OPEN_CHOICES:
|
|
48
|
+
raise KdevError(f"--open takes one of: {', '.join(OPEN_CHOICES)}.")
|
|
49
|
+
cfg = load_ready(interactive)
|
|
50
|
+
target = need_notebook(cfg)
|
|
51
|
+
if not cfg.ssh_public_key:
|
|
52
|
+
raise KdevError("No SSH key configured on this machine.", "Run: kdev setup")
|
|
53
|
+
|
|
54
|
+
gpu, hours, gpu_picked, hours_picked = _box_spec(cfg, gpu, hours, interactive)
|
|
55
|
+
name, prof = quota.pick_account(cfg, account, hours, gpu, interactive)
|
|
56
|
+
ui.header("up", context(cfg, name, target))
|
|
57
|
+
cf = _cloudflared(interactive)
|
|
58
|
+
|
|
59
|
+
# A box already running is the one you left: from this account or another,
|
|
60
|
+
# this machine or another. Connect to it rather than start a second.
|
|
61
|
+
if api.session_status(prof.creds, target).get("status") in api.LIVE_STATES:
|
|
62
|
+
live = ""
|
|
63
|
+
if not replace:
|
|
64
|
+
with ui.spinner("a session is running; checking it is your box…"):
|
|
65
|
+
live = session.find_live_box(cfg, prof.creds, target)
|
|
66
|
+
if live == session.STOPPING:
|
|
67
|
+
# `kdev down` a moment ago: wait for its save, then start afresh.
|
|
68
|
+
# Connecting would find no tunnel; planning now would miss the save.
|
|
69
|
+
with ui.spinner("your last box is still saving your files…"):
|
|
70
|
+
session.wait_saved(prof.creds, target, running_too=True)
|
|
71
|
+
ui.ok("the last box finished saving")
|
|
72
|
+
must_stop = False
|
|
73
|
+
elif live:
|
|
74
|
+
choice = _running_choice(target, interactive)
|
|
75
|
+
if choice == "cancel":
|
|
76
|
+
return
|
|
77
|
+
if choice == "connect":
|
|
78
|
+
_connect(cfg, prof, target, live, cf, hours, gpu, name, open_with, interactive)
|
|
79
|
+
return
|
|
80
|
+
must_stop = True # "replace" was chosen
|
|
81
|
+
elif api.session_status(prof.creds, target).get("status") in api.LIVE_STATES:
|
|
82
|
+
if not replace:
|
|
83
|
+
ui.warn(f"{target} has a session kdev cannot reach (started in the Kaggle editor?)")
|
|
84
|
+
if not (interactive and ui.confirm("Replace it with a kdev box?", default=False)):
|
|
85
|
+
raise KdevError("Left it running.", "Stop it with `kdev down`, then `kdev up`.")
|
|
86
|
+
must_stop = True
|
|
87
|
+
else:
|
|
88
|
+
# It ended while we were looking (finding a box can take minutes):
|
|
89
|
+
# nothing is left to stop, so just start.
|
|
90
|
+
must_stop = False
|
|
91
|
+
# Replacing: stop it and wait for its save *before* planning the
|
|
92
|
+
# restore. The running version has no saved output yet, so planning now
|
|
93
|
+
# would build on the one before it and drop everything done since.
|
|
94
|
+
if must_stop:
|
|
95
|
+
_stop_and_wait(cfg, prof.creds, target)
|
|
96
|
+
|
|
97
|
+
with ui.spinner("reading the notebook…"):
|
|
98
|
+
verdict, existing, backup = safety.check(prof.creds, target)
|
|
99
|
+
if not (cfg.tunnel_credentials or cfg.tunnel_token) and nb.adopt_config(cfg, prof.creds):
|
|
100
|
+
config.save(cfg)
|
|
101
|
+
ui.ok(f"using this workspace's tunnel, {cfg.tunnel_hostname or 'token'}")
|
|
102
|
+
|
|
103
|
+
# Planned even with --no-restore: the box must record what it *should*
|
|
104
|
+
# hold, or a deliberately empty session would look complete and every
|
|
105
|
+
# later `kdev up` would build on it alone, dropping the history.
|
|
106
|
+
with ui.spinner("finding where you left off…"):
|
|
107
|
+
session.wait_saved(prof.creds, target)
|
|
108
|
+
try:
|
|
109
|
+
meta = api.get_kernel(prof.creds, target)
|
|
110
|
+
except api.KaggleError:
|
|
111
|
+
meta = {}
|
|
112
|
+
try:
|
|
113
|
+
latest = int(meta.get("currentVersionNumber") or 0)
|
|
114
|
+
except (TypeError, ValueError):
|
|
115
|
+
latest = 0
|
|
116
|
+
layers = persistence.plan_layers(prof.creds, target, latest)
|
|
117
|
+
# SaveKernel sets the title it is given: keep the notebook's own, or a
|
|
118
|
+
# notebook made in the Kaggle editor ("v0.1.0_testing") gets renamed to
|
|
119
|
+
# its slug ("v0-1-0-testing").
|
|
120
|
+
title = meta.get("title") or target.split("/")[-1]
|
|
121
|
+
if layers and not restore:
|
|
122
|
+
ui.warn(f"starting empty; {' + '.join(layers)} stays saved for next time")
|
|
123
|
+
|
|
124
|
+
if hours > 4 and not cfg.git_remote:
|
|
125
|
+
ui.hint(
|
|
126
|
+
f"Tip: a git remote pushes every 15 min, so even a lost machine costs "
|
|
127
|
+
f"minutes, not {hours:g}h. kdev config set git-remote <url>"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
hold = int(hours * 3600)
|
|
131
|
+
script = bootstrap.render(
|
|
132
|
+
ssh_public_key=cfg.ssh_public_key,
|
|
133
|
+
tunnel_token=cfg.tunnel_token,
|
|
134
|
+
tunnel_credentials=cfg.tunnel_credentials,
|
|
135
|
+
tunnel_id=cfg.tunnel_id,
|
|
136
|
+
tunnel_hostname=cfg.tunnel_hostname,
|
|
137
|
+
# Stop ~2 min before Kaggle does, so the teardown push and the saved
|
|
138
|
+
# version both land.
|
|
139
|
+
hold_seconds=max(hold - 120, 60),
|
|
140
|
+
run_by=prof.username,
|
|
141
|
+
git_remote=cfg.git_remote,
|
|
142
|
+
git_deploy_key=cfg.git_deploy_key,
|
|
143
|
+
restore_layers=layers,
|
|
144
|
+
notebook=target,
|
|
145
|
+
)
|
|
146
|
+
source, kernel_type = bootstrap.inject(existing, script)
|
|
147
|
+
if verdict == "occupied":
|
|
148
|
+
ui.hint(f"your code in the notebook is kept below kdev's cell (backup: {backup})")
|
|
149
|
+
|
|
150
|
+
label = ui.gpu_label(gpu)
|
|
151
|
+
stages = [
|
|
152
|
+
("submit", "Submitting"),
|
|
153
|
+
("queue", "Waiting for a machine"),
|
|
154
|
+
*bootstrap.STAGES,
|
|
155
|
+
("restore", "Restoring your files"),
|
|
156
|
+
("ready", "Ready"),
|
|
157
|
+
]
|
|
158
|
+
host, reachable = None, False
|
|
159
|
+
seen: dict = {}
|
|
160
|
+
restored: dict = {}
|
|
161
|
+
with ui.Stages(stages, head=f"{name} · {label} · {hours:g}h") as board:
|
|
162
|
+
board.advance("submit")
|
|
163
|
+
resp = api.save_kernel(
|
|
164
|
+
prof.creds,
|
|
165
|
+
slug=target,
|
|
166
|
+
title=title,
|
|
167
|
+
source=source,
|
|
168
|
+
machine_shape=api.SHAPES[gpu],
|
|
169
|
+
timeout_seconds=hold,
|
|
170
|
+
kernel_type=kernel_type,
|
|
171
|
+
)
|
|
172
|
+
board.note(f"version {resp.get('versionNumber')}")
|
|
173
|
+
board.advance("queue")
|
|
174
|
+
host = session.await_ready(prof.creds, target, board, seen=seen)
|
|
175
|
+
if host:
|
|
176
|
+
sshcfg.write(cfg.ssh_host_alias, host, cloudflared_path=cf)
|
|
177
|
+
board.advance("restore")
|
|
178
|
+
if layers and restore:
|
|
179
|
+
restored = session.restore(
|
|
180
|
+
prof.creds, target, cfg.ssh_host_alias, layers, board.note
|
|
181
|
+
)
|
|
182
|
+
else:
|
|
183
|
+
board.note("nothing saved yet" if not layers else "skipped (--no-restore)")
|
|
184
|
+
board.advance("ready")
|
|
185
|
+
# The log says the tunnel is up a moment before Cloudflare's edge
|
|
186
|
+
# routes it; ready has to mean an ssh would work right now.
|
|
187
|
+
board.note("waiting for ssh…")
|
|
188
|
+
reachable = persistence.wait_reachable(cfg.ssh_host_alias)
|
|
189
|
+
if reachable:
|
|
190
|
+
board.note("")
|
|
191
|
+
board.finish()
|
|
192
|
+
else:
|
|
193
|
+
board.fail("the tunnel is up but ssh does not answer")
|
|
194
|
+
else:
|
|
195
|
+
board.fail("no tunnel was announced")
|
|
196
|
+
|
|
197
|
+
if not host:
|
|
198
|
+
_explain_no_tunnel(prof.creds, target, seen)
|
|
199
|
+
# Only what was answered at a prompt is remembered, field by field: an
|
|
200
|
+
# explicit --gpu for this run must not ride along into the defaults just
|
|
201
|
+
# because the session length happened to be asked for.
|
|
202
|
+
if gpu_picked:
|
|
203
|
+
cfg.default_gpu = gpu
|
|
204
|
+
if hours_picked:
|
|
205
|
+
cfg.default_hours = hours
|
|
206
|
+
if gpu_picked or hours_picked:
|
|
207
|
+
config.save(cfg)
|
|
208
|
+
session.report_restore(restored)
|
|
209
|
+
if not reachable:
|
|
210
|
+
raise KdevError(
|
|
211
|
+
f"The box is running at {host}, but ssh does not answer yet.",
|
|
212
|
+
"Try again in a minute: kdev ssh See what it is doing: kdev logs",
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
ui.blank()
|
|
216
|
+
files = f"{restored.get('done', 0)} restored" if restored.get("total") else ""
|
|
217
|
+
ui.ready_card(cfg.ssh_host_alias, host, hours, label, name, note=files)
|
|
218
|
+
_open(cfg.ssh_host_alias, open_with, interactive)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _box_spec(
|
|
222
|
+
cfg: config.Config, gpu: str | None, hours: float | None, interactive: bool
|
|
223
|
+
) -> tuple[str, float, bool, bool]:
|
|
224
|
+
"""Flag, then the saved default, then ask. Returns (gpu, hours, and
|
|
225
|
+
whether each was answered at a prompt). Only prompted answers become
|
|
226
|
+
defaults: `kdev up --gpu none` for a quick CPU job must not quietly make
|
|
227
|
+
every later `kdev up` a CPU box."""
|
|
228
|
+
gpu_picked = hours_picked = False
|
|
229
|
+
if gpu is None:
|
|
230
|
+
if cfg.default_gpu:
|
|
231
|
+
gpu = cfg.default_gpu
|
|
232
|
+
elif interactive:
|
|
233
|
+
gpu, gpu_picked = ui.pick_gpu(), True
|
|
234
|
+
else:
|
|
235
|
+
gpu = "t4"
|
|
236
|
+
if gpu not in api.SHAPES:
|
|
237
|
+
raise KdevError(f"--gpu must be one of: {', '.join(api.SHAPES)}.")
|
|
238
|
+
cap = cap_for(gpu)
|
|
239
|
+
if hours is None:
|
|
240
|
+
if cfg.default_hours:
|
|
241
|
+
hours = cfg.default_hours
|
|
242
|
+
elif interactive:
|
|
243
|
+
hours, hours_picked = ui.pick_hours(gpu, cap), True
|
|
244
|
+
else:
|
|
245
|
+
hours = 9.0
|
|
246
|
+
if not hours or hours <= 0:
|
|
247
|
+
raise KdevError("--hours must be more than zero.")
|
|
248
|
+
if hours > cap:
|
|
249
|
+
ui.warn(f"Kaggle stops {ui.gpu_label(gpu)} sessions at {cap:g}h; using {cap:g}h")
|
|
250
|
+
hours = cap
|
|
251
|
+
return gpu, float(hours), gpu_picked, hours_picked
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _cloudflared(interactive: bool):
|
|
255
|
+
"""Resolved before the session starts: discovering it is missing after the
|
|
256
|
+
container boots would waste quota on a box nobody can reach."""
|
|
257
|
+
cf = cloudflared.find()
|
|
258
|
+
if cf:
|
|
259
|
+
return cf
|
|
260
|
+
if interactive and not ui.confirm("cloudflared is missing (ssh needs it). Download it now?"):
|
|
261
|
+
raise KdevError(
|
|
262
|
+
"kdev needs cloudflared to reach the box.",
|
|
263
|
+
"Install it: brew install cloudflared or: kdev doctor --fix",
|
|
264
|
+
)
|
|
265
|
+
cf = ui.download("cloudflared", cloudflared.install)
|
|
266
|
+
ui.ok(f"cloudflared {cloudflared.version(cf)}")
|
|
267
|
+
return cf
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _stop_and_wait(
|
|
271
|
+
cfg: config.Config, creds: api.Creds, target: str, timeout: int = session.SAVE_TIMEOUT
|
|
272
|
+
) -> None:
|
|
273
|
+
with ui.spinner("stopping the running box and waiting for it to save…"):
|
|
274
|
+
stopped = False
|
|
275
|
+
if session.reachable(cfg.ssh_host_alias):
|
|
276
|
+
r = subprocess.run(
|
|
277
|
+
[
|
|
278
|
+
"ssh",
|
|
279
|
+
"-T",
|
|
280
|
+
"-o",
|
|
281
|
+
"BatchMode=yes",
|
|
282
|
+
cfg.ssh_host_alias,
|
|
283
|
+
"touch /kaggle/working/.kdev-stop",
|
|
284
|
+
],
|
|
285
|
+
capture_output=True,
|
|
286
|
+
)
|
|
287
|
+
stopped = r.returncode == 0
|
|
288
|
+
if not stopped:
|
|
289
|
+
sid, runner = session.live_session_id(creds, target)
|
|
290
|
+
if not sid or not session.cancel_as_owner(cfg, int(sid), runner):
|
|
291
|
+
raise KdevError(
|
|
292
|
+
"Could not stop the running box from here.",
|
|
293
|
+
f"Stop it at {nb.notebook_url(target)}, then run kdev up again.",
|
|
294
|
+
)
|
|
295
|
+
deadline = time.time() + timeout
|
|
296
|
+
while api.session_status(creds, target).get("status") in api.LIVE_STATES:
|
|
297
|
+
if time.time() > deadline:
|
|
298
|
+
raise KdevError(
|
|
299
|
+
"The old box is taking too long to stop.",
|
|
300
|
+
"Check it with kdev status, then run kdev up again.",
|
|
301
|
+
)
|
|
302
|
+
time.sleep(5)
|
|
303
|
+
ui.ok("stopped the old box; its files are saved")
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _running_choice(target: str, interactive: bool) -> str:
|
|
307
|
+
"""connect, replace or cancel, for a box that is already up."""
|
|
308
|
+
ui.ok(f"{target} is already running")
|
|
309
|
+
if not interactive:
|
|
310
|
+
return "connect"
|
|
311
|
+
return ui.choose(
|
|
312
|
+
"It is your box. What now?",
|
|
313
|
+
[
|
|
314
|
+
("connect", "Connect to it (recommended)"),
|
|
315
|
+
("replace", "Stop it and start a new one"),
|
|
316
|
+
("cancel", "Nothing"),
|
|
317
|
+
],
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _connect(cfg, prof, target, host, cf, hours, gpu, account, open_with, interactive) -> None:
|
|
322
|
+
sshcfg.write(cfg.ssh_host_alias, host, cloudflared_path=cf)
|
|
323
|
+
with ui.spinner("waiting for ssh…"):
|
|
324
|
+
ok = persistence.wait_reachable(cfg.ssh_host_alias, timeout=60)
|
|
325
|
+
if not ok:
|
|
326
|
+
raise KdevError(
|
|
327
|
+
"The box is running but ssh does not answer.",
|
|
328
|
+
"If this laptop has never started this workspace, its key is not on "
|
|
329
|
+
"the box yet; it is added the first time it runs `kdev up`.",
|
|
330
|
+
)
|
|
331
|
+
state = persistence.box_state(cfg.ssh_host_alias)
|
|
332
|
+
if state and not state.get("restored"):
|
|
333
|
+
ui.warn("That session never finished restoring your files", "Finish it: kdev restore")
|
|
334
|
+
hours, label, account = session.describe_running(cfg, prof.creds, target, hours, gpu, account)
|
|
335
|
+
ui.blank()
|
|
336
|
+
ui.ready_card(cfg.ssh_host_alias, host, hours, label, account)
|
|
337
|
+
_open(cfg.ssh_host_alias, open_with, interactive)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _open(alias: str, how: str | None, interactive: bool) -> None:
|
|
341
|
+
"""Open the box the way you want to work in it."""
|
|
342
|
+
if how is None:
|
|
343
|
+
if not interactive:
|
|
344
|
+
return
|
|
345
|
+
editor = session.editor()
|
|
346
|
+
options = [("code", f"Open in {editor} (recommended)")] if editor else []
|
|
347
|
+
options += [("shell", "Open a shell here"), ("none", "Leave it running")]
|
|
348
|
+
how = ui.choose("Open it?", options)
|
|
349
|
+
if how == "code":
|
|
350
|
+
session.open_editor(alias)
|
|
351
|
+
elif how == "shell":
|
|
352
|
+
session.open_shell(alias)
|
|
353
|
+
ui.hint("Back on your machine. The box is still running: kdev down stops it.")
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _explain_no_tunnel(creds: api.Creds, target: str, seen: dict) -> None:
|
|
357
|
+
try:
|
|
358
|
+
state = api.session_status(creds, target)
|
|
359
|
+
except api.KaggleError:
|
|
360
|
+
state = {}
|
|
361
|
+
hint = "See what it did: kdev logs"
|
|
362
|
+
if state.get("status") in api.LIVE_STATES:
|
|
363
|
+
ui.warn(f"It is still {state['status'].lower()} and still using quota")
|
|
364
|
+
hint = (
|
|
365
|
+
f"Stop it: kdev down --session-id {seen['session']}"
|
|
366
|
+
if seen.get("session")
|
|
367
|
+
else f"Stop it at {nb.notebook_url(target)}"
|
|
368
|
+
)
|
|
369
|
+
if state.get("failureMessage"):
|
|
370
|
+
ui.hint(state["failureMessage"])
|
|
371
|
+
raise KdevError("The session never reported a tunnel.", hint)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# --- down ---------------------------------------------------------------------
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def down(
|
|
378
|
+
session_id: int = typer.Option(0, "--session-id", help="Cancel this session id directly."),
|
|
379
|
+
account: str = typer.Option("", "--account", "-a", help="Cancel as this account."),
|
|
380
|
+
) -> None:
|
|
381
|
+
"""Stop the box. Your files are saved however it stops."""
|
|
382
|
+
cfg = config.load()
|
|
383
|
+
session.ssh_ready(cfg)
|
|
384
|
+
|
|
385
|
+
if session_id:
|
|
386
|
+
if account:
|
|
387
|
+
api.cancel_session(cfg.profile(account).creds, session_id)
|
|
388
|
+
by = account
|
|
389
|
+
else:
|
|
390
|
+
by = session.cancel_as_owner(cfg, session_id)
|
|
391
|
+
if not by:
|
|
392
|
+
raise KdevError(
|
|
393
|
+
f"None of the signed-in accounts may cancel {session_id}.",
|
|
394
|
+
"Kaggle only lets the account that started it do that.",
|
|
395
|
+
)
|
|
396
|
+
session.box_gone(cfg)
|
|
397
|
+
ui.ok(f"cancelled session {session_id} as {by}; Kaggle saves its files as it stops")
|
|
398
|
+
return
|
|
399
|
+
|
|
400
|
+
alias = cfg.ssh_host_alias
|
|
401
|
+
try:
|
|
402
|
+
r = subprocess.run(
|
|
403
|
+
[
|
|
404
|
+
"ssh",
|
|
405
|
+
"-T",
|
|
406
|
+
"-o",
|
|
407
|
+
"BatchMode=yes",
|
|
408
|
+
"-o",
|
|
409
|
+
"ConnectTimeout=15",
|
|
410
|
+
alias,
|
|
411
|
+
"touch /kaggle/working/.kdev-stop",
|
|
412
|
+
],
|
|
413
|
+
capture_output=True,
|
|
414
|
+
text=True,
|
|
415
|
+
timeout=60,
|
|
416
|
+
)
|
|
417
|
+
reached = r.returncode == 0
|
|
418
|
+
except (subprocess.SubprocessError, OSError):
|
|
419
|
+
reached = False
|
|
420
|
+
if reached:
|
|
421
|
+
session.box_gone(cfg)
|
|
422
|
+
ui.ok(
|
|
423
|
+
"stopping; the box saves /kaggle/working as it exits",
|
|
424
|
+
"Next `kdev up` brings it all back, on any account or machine.",
|
|
425
|
+
)
|
|
426
|
+
return
|
|
427
|
+
|
|
428
|
+
# It does not answer. It may be gone, or unreachable while still using
|
|
429
|
+
# quota: find out, and cancel it as whoever started it.
|
|
430
|
+
target = need_notebook(cfg)
|
|
431
|
+
creds = cfg.profile(account).creds
|
|
432
|
+
status = api.session_status(creds, target).get("status", "?")
|
|
433
|
+
if status not in api.LIVE_STATES:
|
|
434
|
+
session.box_gone(cfg)
|
|
435
|
+
ui.ok(f"nothing running ({status.lower()}); your files are in the saved version")
|
|
436
|
+
return
|
|
437
|
+
if session.stopping(creds, target):
|
|
438
|
+
session.box_gone(cfg)
|
|
439
|
+
ui.ok(
|
|
440
|
+
"already stopping; Kaggle is saving /kaggle/working",
|
|
441
|
+
"Next `kdev up` waits for the save, then brings it all back.",
|
|
442
|
+
)
|
|
443
|
+
return
|
|
444
|
+
with ui.spinner("the box does not answer; finding its session id…"):
|
|
445
|
+
sid, runner = session.live_session_id(creds, target)
|
|
446
|
+
if not sid:
|
|
447
|
+
raise KdevError(
|
|
448
|
+
"The box is still running but does not answer, and its id is not in the log.",
|
|
449
|
+
f"Stop it at {nb.notebook_url(target)} (files are saved either way)",
|
|
450
|
+
)
|
|
451
|
+
by = session.cancel_as_owner(cfg, int(sid), runner)
|
|
452
|
+
if not by:
|
|
453
|
+
raise KdevError(
|
|
454
|
+
f"Only {runner or 'the account that started it'} can cancel session "
|
|
455
|
+
f"{sid}, and it is not signed in here.",
|
|
456
|
+
f"Sign it in (kdev account add) or stop it at {nb.notebook_url(target)}",
|
|
457
|
+
)
|
|
458
|
+
session.box_gone(cfg)
|
|
459
|
+
ui.ok(f"cancelled session {sid} as {by}; Kaggle saves its files as it stops")
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
# --- ssh ----------------------------------------------------------------------
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def ssh(ctx: typer.Context) -> None:
|
|
466
|
+
"""Open a shell on the box. Anything after -- goes to ssh."""
|
|
467
|
+
cfg = config.load()
|
|
468
|
+
if not cfg.ssh_host_alias:
|
|
469
|
+
raise KdevError("No ssh alias configured.", "Run: kdev setup")
|
|
470
|
+
session.ssh_ready(cfg)
|
|
471
|
+
if not sshcfg.has_block():
|
|
472
|
+
raise KdevError("No box hostname known yet.", "Start or find it with: kdev up")
|
|
473
|
+
# ssh's own exit code is this command's, so scripts can wrap it.
|
|
474
|
+
raise typer.Exit(subprocess.run(["ssh", cfg.ssh_host_alias, *ctx.args]).returncode)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
# --- status -------------------------------------------------------------------
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def status(
|
|
481
|
+
account: str = typer.Option("", "--account", "-a", help="Check as this account."),
|
|
482
|
+
as_json: bool = typer.Option(False, "--json", help="Print JSON instead."),
|
|
483
|
+
) -> None:
|
|
484
|
+
"""The box: running or not, who started it, when it ends, your files."""
|
|
485
|
+
cfg = config.load()
|
|
486
|
+
session.ssh_ready(cfg)
|
|
487
|
+
target = need_notebook(cfg)
|
|
488
|
+
creds = cfg.profile(account).creds
|
|
489
|
+
with ui.spinner("checking…"):
|
|
490
|
+
try:
|
|
491
|
+
state = api.session_status(creds, target)
|
|
492
|
+
except api.KaggleError as e:
|
|
493
|
+
state = {"status": "UNKNOWN", "failureMessage": str(e)}
|
|
494
|
+
reachable = session.reachable(cfg.ssh_host_alias)
|
|
495
|
+
box = persistence.box_state(cfg.ssh_host_alias) if reachable else {}
|
|
496
|
+
running = state.get("status") in api.LIVE_STATES
|
|
497
|
+
saving = running and not reachable and session.stopping(creds, target)
|
|
498
|
+
info = {
|
|
499
|
+
"notebook": target,
|
|
500
|
+
"status": state.get("status", "?"),
|
|
501
|
+
"running": running,
|
|
502
|
+
"stopping": saving,
|
|
503
|
+
"reachable": reachable,
|
|
504
|
+
"host": cfg.tunnel_hostname or None,
|
|
505
|
+
"started_by": box.get("run_by") or None,
|
|
506
|
+
"session": box.get("session") or None,
|
|
507
|
+
"ends": box.get("ends") or None,
|
|
508
|
+
"restored": box.get("restored") if box else None,
|
|
509
|
+
"restored_from": box.get("layers") if box else None,
|
|
510
|
+
"failure": state.get("failureMessage") or None,
|
|
511
|
+
}
|
|
512
|
+
if as_json:
|
|
513
|
+
ui.emit_json(info)
|
|
514
|
+
return
|
|
515
|
+
|
|
516
|
+
# Kaggle's words describe the last *run*; the user asked about the *box*.
|
|
517
|
+
word = {"RUNNING": "running", "QUEUED": "starting"}.get(info["status"], "stopped")
|
|
518
|
+
if saving:
|
|
519
|
+
word = "stopping"
|
|
520
|
+
title = Text.assemble(
|
|
521
|
+
(
|
|
522
|
+
ui.g("live") + " " if running else ui.g("pending") + " ",
|
|
523
|
+
"kdev.ok" if running else "kdev.muted",
|
|
524
|
+
),
|
|
525
|
+
(word, "bold"),
|
|
526
|
+
(" " + target, "kdev.muted"),
|
|
527
|
+
)
|
|
528
|
+
rows: list[tuple[str, object]] = []
|
|
529
|
+
if not running:
|
|
530
|
+
last = {
|
|
531
|
+
"COMPLETE": "finished",
|
|
532
|
+
"ERROR": "ended with an error",
|
|
533
|
+
"CANCEL_ACKNOWLEDGED": "was cancelled",
|
|
534
|
+
"CANCEL_REQUESTED": "is being cancelled",
|
|
535
|
+
api.NEVER_RUN: "never run yet",
|
|
536
|
+
}
|
|
537
|
+
rows.append(("last run", last.get(info["status"], info["status"].lower())))
|
|
538
|
+
if info["status"] == api.NEVER_RUN:
|
|
539
|
+
rows.append(("files", "nothing saved yet; `kdev up` starts it"))
|
|
540
|
+
else:
|
|
541
|
+
rows.append(("files", "saved in the notebook; `kdev up` puts them back"))
|
|
542
|
+
if box.get("run_by"):
|
|
543
|
+
rows.append(("started by", box["run_by"]))
|
|
544
|
+
if box.get("ends"):
|
|
545
|
+
left = max(0, int(box["ends"]) - int(time.time()))
|
|
546
|
+
ends = time.strftime("%H:%M", time.localtime(int(box["ends"])))
|
|
547
|
+
rows.append(("ends", f"{ends} ({ui.fmt_hours(left)} left)"))
|
|
548
|
+
if saving:
|
|
549
|
+
rows.append(("files", "being saved; `kdev up` waits for them"))
|
|
550
|
+
elif running:
|
|
551
|
+
reach = Text(
|
|
552
|
+
f"{cfg.ssh_host_alias} {ui.g('arrow')} {cfg.tunnel_hostname or 'quick tunnel'} "
|
|
553
|
+
)
|
|
554
|
+
reach.append(
|
|
555
|
+
"reachable" if reachable else "not reachable yet",
|
|
556
|
+
style="kdev.ok" if reachable else "kdev.warn",
|
|
557
|
+
)
|
|
558
|
+
rows.append(("ssh", reach))
|
|
559
|
+
if box:
|
|
560
|
+
layers = " + ".join(box.get("layers") or []) or "nothing saved yet"
|
|
561
|
+
rows.append(
|
|
562
|
+
(
|
|
563
|
+
"files",
|
|
564
|
+
Text(f"restored from {layers}", style="kdev.ok")
|
|
565
|
+
if box.get("restored")
|
|
566
|
+
else Text("restore not finished: kdev restore", style="kdev.warn"),
|
|
567
|
+
)
|
|
568
|
+
)
|
|
569
|
+
if box.get("notebook") and box["notebook"] != target:
|
|
570
|
+
rows.append(
|
|
571
|
+
("note", Text(f"the running box is {box['notebook']}, not {target}", style="kdev.warn"))
|
|
572
|
+
)
|
|
573
|
+
if info["failure"]:
|
|
574
|
+
rows.append(("error", Text(info["failure"], style="kdev.err")))
|
|
575
|
+
ui.card(title, rows, tone="ok" if running else "muted")
|
|
576
|
+
if running and not reachable and not saving:
|
|
577
|
+
ui.hint("Running but not reachable yet: give it a minute, or `kdev logs` to see why.")
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
# --- logs ---------------------------------------------------------------------
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def logs(
|
|
584
|
+
account: str = typer.Option("", "--account", "-a", help="Read as this account."),
|
|
585
|
+
lines: int = typer.Option(0, "--lines", "-n", help="Stop after N lines (default: follow)."),
|
|
586
|
+
everything: bool = typer.Option(False, "--all", help="Include kdev's heartbeat lines."),
|
|
587
|
+
) -> None:
|
|
588
|
+
"""Stream the session's log."""
|
|
589
|
+
cfg = config.load()
|
|
590
|
+
target = need_notebook(cfg)
|
|
591
|
+
shown = 0
|
|
592
|
+
for line in api.stream_logs(cfg.profile(account).creds, target):
|
|
593
|
+
text = line.rstrip()
|
|
594
|
+
if not everything and text.startswith("KDEV_ALIVE"):
|
|
595
|
+
continue
|
|
596
|
+
ui.console.print(text, highlight=False, markup=False)
|
|
597
|
+
shown += 1
|
|
598
|
+
if lines and shown >= lines:
|
|
599
|
+
return
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
# --- restore ------------------------------------------------------------------
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def restore_cmd(
|
|
606
|
+
account: str = typer.Option("", "--account", "-a", help="Read the notebook as this account."),
|
|
607
|
+
from_version: str = typer.Option(
|
|
608
|
+
"", "--from", help="Bring back files from this saved version, e.g. v12."
|
|
609
|
+
),
|
|
610
|
+
from_backup: bool = typer.Option(
|
|
611
|
+
False, "--from-backup", help="Push this machine's backup instead."
|
|
612
|
+
),
|
|
613
|
+
) -> None:
|
|
614
|
+
"""Put your saved files back into the running box.
|
|
615
|
+
|
|
616
|
+
`kdev up` already does this. Run it to finish a restore that was cut
|
|
617
|
+
short, bring back files that an older version still has, or push a local
|
|
618
|
+
backup. It only adds what is missing: a file on the box now is never
|
|
619
|
+
overwritten, so this is not a rollback.
|
|
620
|
+
"""
|
|
621
|
+
cfg = config.load()
|
|
622
|
+
session.ssh_ready(cfg)
|
|
623
|
+
alias = cfg.ssh_host_alias
|
|
624
|
+
if from_backup:
|
|
625
|
+
files = persistence.contents()
|
|
626
|
+
if not files:
|
|
627
|
+
raise KdevError("This machine has no backup yet.", "Make one with: kdev backup")
|
|
628
|
+
with ui.spinner("pushing the backup…"):
|
|
629
|
+
ok, err = persistence.push(alias)
|
|
630
|
+
if not ok:
|
|
631
|
+
raise KdevError(
|
|
632
|
+
f"Could not push the backup: {err or 'ssh failed'}", "Is the box up? kdev status"
|
|
633
|
+
)
|
|
634
|
+
ui.ok(f"pushed {len(files)} file(s) into /kaggle/working")
|
|
635
|
+
return
|
|
636
|
+
|
|
637
|
+
if from_version and not re.fullmatch(r"v\d+", from_version):
|
|
638
|
+
raise KdevError("--from takes a version like v12.", "See them: kdev workspace files")
|
|
639
|
+
creds = cfg.profile(account).creds
|
|
640
|
+
with ui.spinner("reading the box…"):
|
|
641
|
+
state = persistence.box_state(alias)
|
|
642
|
+
if not state:
|
|
643
|
+
raise KdevError("Could not read the box over ssh.", "Is it up? kdev status")
|
|
644
|
+
# Version labels only mean something for the notebook that saved them: use
|
|
645
|
+
# the box's own, not whatever this machine is pointed at now.
|
|
646
|
+
target = state.get("notebook") or need_notebook(cfg)
|
|
647
|
+
if target != cfg.notebook:
|
|
648
|
+
ui.warn(f"the running box belongs to {target}; restoring from that notebook")
|
|
649
|
+
if from_version and not persistence.version_files(creds, target, from_version):
|
|
650
|
+
raise KdevError(
|
|
651
|
+
f"{from_version} of {target} has no saved files.",
|
|
652
|
+
"A running box's own version has none yet. See one: kdev workspace files --version vN",
|
|
653
|
+
)
|
|
654
|
+
layers = [from_version] if from_version else list(state.get("layers") or [])
|
|
655
|
+
if not layers:
|
|
656
|
+
with ui.spinner("finding where you left off…"):
|
|
657
|
+
latest = int(api.get_kernel(creds, target).get("currentVersionNumber") or 0)
|
|
658
|
+
# The running session is the newest version; build on the one before.
|
|
659
|
+
layers = persistence.plan_layers(creds, target, latest - 1)
|
|
660
|
+
if not layers:
|
|
661
|
+
ui.ok("nothing saved yet, so nothing to restore")
|
|
662
|
+
return
|
|
663
|
+
if state.get("restored") and not from_version:
|
|
664
|
+
ui.hint("already restored; filling in anything that is missing")
|
|
665
|
+
ui.ok(f"restoring from {' + '.join(layers)}")
|
|
666
|
+
with ui.spinner("restoring…") as spin:
|
|
667
|
+
result = session.restore(
|
|
668
|
+
creds,
|
|
669
|
+
target,
|
|
670
|
+
alias,
|
|
671
|
+
layers,
|
|
672
|
+
lambda text: spin.update(Text(f"restoring… {text}", style="kdev.muted")),
|
|
673
|
+
)
|
|
674
|
+
session.report_restore(result)
|
|
675
|
+
if not result.get("restored"):
|
|
676
|
+
raise typer.Exit(1)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
# --- backup -------------------------------------------------------------------
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def backup() -> None:
|
|
683
|
+
"""Copy the running box's /kaggle/working to this machine.
|
|
684
|
+
|
|
685
|
+
Optional: Kaggle keeps your files anyway. This is a local copy you can
|
|
686
|
+
open in Finder, and push back with `kdev restore --from-backup`.
|
|
687
|
+
"""
|
|
688
|
+
cfg = config.load()
|
|
689
|
+
session.ssh_ready(cfg)
|
|
690
|
+
with ui.spinner("copying the box to this machine…"):
|
|
691
|
+
ok, err = persistence.pull(cfg.ssh_host_alias)
|
|
692
|
+
if not ok:
|
|
693
|
+
raise KdevError(f"Could not reach the box over ssh. {err}".strip(), "Is it up? kdev status")
|
|
694
|
+
ui.ok(f"{len(persistence.contents())} file(s) in {persistence.mirror_dir()}")
|