kdev-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kdev/__init__.py +7 -0
- kdev/__main__.py +3 -0
- kdev/api.py +408 -0
- kdev/auth.py +115 -0
- kdev/bootstrap.py +510 -0
- kdev/cli.py +162 -0
- kdev/cloudflared.py +179 -0
- kdev/commands/__init__.py +5 -0
- kdev/commands/_common.py +51 -0
- kdev/commands/account.py +148 -0
- kdev/commands/box.py +694 -0
- kdev/commands/doctor.py +127 -0
- kdev/commands/home.py +92 -0
- kdev/commands/settings.py +164 -0
- kdev/commands/tunnel.py +78 -0
- kdev/commands/workspace.py +210 -0
- kdev/config.py +181 -0
- kdev/errors.py +25 -0
- kdev/kdev_box.py +538 -0
- kdev/notebook.py +261 -0
- kdev/persistence.py +314 -0
- kdev/quota.py +102 -0
- kdev/safety.py +72 -0
- kdev/session.py +284 -0
- kdev/sshcfg.py +72 -0
- kdev/tunnel.py +259 -0
- kdev/ui.py +666 -0
- kdev/wizard.py +251 -0
- kdev_cli-0.1.3.dist-info/METADATA +598 -0
- kdev_cli-0.1.3.dist-info/RECORD +33 -0
- kdev_cli-0.1.3.dist-info/WHEEL +4 -0
- kdev_cli-0.1.3.dist-info/entry_points.txt +2 -0
- kdev_cli-0.1.3.dist-info/licenses/LICENSE +21 -0
kdev/bootstrap.py
ADDED
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
"""Generates the Python script that Kaggle runs as the notebook body.
|
|
2
|
+
|
|
3
|
+
It is a *script* kernel, not an .ipynb: the remote side only has to boot a
|
|
4
|
+
tunnel and block, so there is nothing a notebook buys us over plain source.
|
|
5
|
+
Config is injected as a JSON literal because Kaggle has no API for Secrets --
|
|
6
|
+
the notebook is private and group-shared, so the blast radius is the group.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import ast
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
MARKER = "# Generated by kdev"
|
|
17
|
+
READY_MARKER = "KDEV_READY"
|
|
18
|
+
STAGE_MARKER = "KDEV_STAGE"
|
|
19
|
+
|
|
20
|
+
#: Ordered stages the remote script announces, for the local progress display.
|
|
21
|
+
STAGES = [
|
|
22
|
+
("sshd", "Installing SSH server"),
|
|
23
|
+
("env", "Exporting notebook environment"),
|
|
24
|
+
("repo", "Syncing workspace"),
|
|
25
|
+
("tunnel", "Starting Cloudflare tunnel"),
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
_TEMPLATE = r'''# Generated by kdev -- do not edit here, edit your kdev config and re-run.
|
|
29
|
+
import json, os, pathlib, shutil, signal, subprocess, sys, time
|
|
30
|
+
|
|
31
|
+
CFG = json.loads(r"""__CONFIG_JSON__""")
|
|
32
|
+
READY = "__READY_MARKER__"
|
|
33
|
+
STAGE = "__STAGE_MARKER__"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def stage(name):
|
|
37
|
+
print(f"{STAGE} {name}", flush=True)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def sh(cmd, check=True, **kw):
|
|
41
|
+
print(f"$ {cmd}", flush=True)
|
|
42
|
+
return subprocess.run(cmd, shell=True, check=check, **kw)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def install_box():
|
|
46
|
+
"""Put kdev's on-box module in place and import it.
|
|
47
|
+
|
|
48
|
+
It arrives inside CFG so the box has it from the first second, without
|
|
49
|
+
waiting on a laptop that may already have gone to sleep.
|
|
50
|
+
"""
|
|
51
|
+
run = pathlib.Path("/root/.kdev")
|
|
52
|
+
run.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
(run / "kdev_box.py").write_text(CFG["box_script"])
|
|
54
|
+
launcher = run / "run"
|
|
55
|
+
launcher.write_text(f"#!/bin/sh\nexec {sys.executable} /root/.kdev/kdev_box.py \"$@\"\n")
|
|
56
|
+
launcher.chmod(0o755)
|
|
57
|
+
sys.path.insert(0, str(run))
|
|
58
|
+
import kdev_box
|
|
59
|
+
return kdev_box
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def session_id():
|
|
63
|
+
"""Kaggle puts the session id in the container name; kdev needs it to
|
|
64
|
+
cancel a box it can no longer reach over ssh."""
|
|
65
|
+
name = os.environ.get("KAGGLE_CONTAINER_NAME", "")
|
|
66
|
+
parts = name.rsplit("-", 2)
|
|
67
|
+
return parts[1] if len(parts) == 3 and parts[1].isdigit() else ""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def install_sshd(box):
|
|
71
|
+
if not shutil.which("sshd"):
|
|
72
|
+
# check=False: a transient apt mirror failure must not end the session
|
|
73
|
+
# before we have even said why. The explicit check below is the real one.
|
|
74
|
+
sh("apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y openssh-server",
|
|
75
|
+
check=False)
|
|
76
|
+
if not (shutil.which("sshd") or pathlib.Path("/usr/sbin/sshd").exists()):
|
|
77
|
+
raise SystemExit("could not install openssh-server; see the apt output above")
|
|
78
|
+
pathlib.Path("/var/run/sshd").mkdir(parents=True, exist_ok=True)
|
|
79
|
+
|
|
80
|
+
# The key of the machine that started this session, plus every key saved
|
|
81
|
+
# in the workspace -- so any machine that has ever used it can get in.
|
|
82
|
+
box.install_keys(CFG["ssh_public_key"])
|
|
83
|
+
|
|
84
|
+
# Password auth stays off: the only way in is the key you already hold.
|
|
85
|
+
pathlib.Path("/etc/ssh/sshd_config.d/kdev.conf").parent.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
pathlib.Path("/etc/ssh/sshd_config.d/kdev.conf").write_text(
|
|
87
|
+
"PermitRootLogin prohibit-password\n"
|
|
88
|
+
"PermitUserEnvironment yes\n"
|
|
89
|
+
"PasswordAuthentication no\n"
|
|
90
|
+
"PubkeyAuthentication yes\n"
|
|
91
|
+
"ClientAliveInterval 30\n"
|
|
92
|
+
"ClientAliveCountMax 240\n"
|
|
93
|
+
"AcceptEnv *\n"
|
|
94
|
+
)
|
|
95
|
+
sh("ssh-keygen -A")
|
|
96
|
+
sh("service ssh restart || /usr/sbin/sshd")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def export_env():
|
|
100
|
+
"""Give every ssh session the notebook's environment -- not only terminals.
|
|
101
|
+
|
|
102
|
+
sshd starts sessions with a bare environment: no conda, no CUDA, no Kaggle
|
|
103
|
+
variables. A hook in .bashrc only helps interactive shells (Ubuntu's .bashrc
|
|
104
|
+
returns early otherwise), so `ssh kaggle python train.py`, VS Code's server
|
|
105
|
+
and its extensions all got a PATH without the GPU tools -- measured:
|
|
106
|
+
`nvidia-smi: command not found` on a T4 box. ~/.ssh/environment is read by
|
|
107
|
+
sshd itself for every session (PermitUserEnvironment is on in kdev.conf).
|
|
108
|
+
"""
|
|
109
|
+
skip = {"LS_COLORS", "_", "PWD", "OLDPWD", "SHLVL", "TERM",
|
|
110
|
+
"SSH_CLIENT", "SSH_CONNECTION", "SSH_TTY"}
|
|
111
|
+
env = {k: v for k, v in sorted(os.environ.items()) if k not in skip and "\n" not in v}
|
|
112
|
+
ssh_env = pathlib.Path("/root/.ssh/environment")
|
|
113
|
+
ssh_env.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
114
|
+
ssh_env.write_text("".join(f"{k}={v}\n" for k, v in env.items()))
|
|
115
|
+
ssh_env.chmod(0o600)
|
|
116
|
+
|
|
117
|
+
# Also kept as a sourceable file, for scripts that start from a clean env.
|
|
118
|
+
pathlib.Path("/root/.kdev_env.sh").write_text(
|
|
119
|
+
"".join(f"export {k}={json.dumps(v)}\n" for k, v in env.items()))
|
|
120
|
+
bashrc = pathlib.Path("/root/.bashrc")
|
|
121
|
+
hook = "\n[ -f /root/.kdev_env.sh ] && . /root/.kdev_env.sh\ncd /kaggle/working\n"
|
|
122
|
+
if "kdev_env" not in (bashrc.read_text() if bashrc.exists() else ""):
|
|
123
|
+
with bashrc.open("a") as f:
|
|
124
|
+
f.write(hook)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def add_env(name, value):
|
|
128
|
+
"""Add one variable to every future ssh session."""
|
|
129
|
+
os.environ[name] = value
|
|
130
|
+
with pathlib.Path("/root/.ssh/environment").open("a") as f:
|
|
131
|
+
f.write(f"{name}={value}\n")
|
|
132
|
+
with pathlib.Path("/root/.kdev_env.sh").open("a") as f:
|
|
133
|
+
f.write(f"export {name}={json.dumps(value)}\n")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def install_cloudflared():
|
|
137
|
+
if shutil.which("cloudflared"):
|
|
138
|
+
return "cloudflared"
|
|
139
|
+
url = ("https://github.com/cloudflare/cloudflared/releases/latest/download/"
|
|
140
|
+
"cloudflared-linux-amd64")
|
|
141
|
+
sh(f"curl -fsSL {url} -o /usr/local/bin/cloudflared && chmod +x /usr/local/bin/cloudflared")
|
|
142
|
+
return "/usr/local/bin/cloudflared"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def start_tunnel(binary):
|
|
146
|
+
pathlib.Path("/kaggle/working").mkdir(parents=True, exist_ok=True)
|
|
147
|
+
log = open("/kaggle/working/cloudflared.log", "w")
|
|
148
|
+
host = CFG.get("tunnel_hostname") or None
|
|
149
|
+
|
|
150
|
+
if CFG.get("tunnel_credentials"):
|
|
151
|
+
# Locally-managed: ingress comes from the config we write right here, so
|
|
152
|
+
# nothing has to be provisioned in the Zero Trust dashboard.
|
|
153
|
+
cf_dir = pathlib.Path("/root/.cloudflared")
|
|
154
|
+
cf_dir.mkdir(parents=True, exist_ok=True)
|
|
155
|
+
creds = cf_dir / "creds.json"
|
|
156
|
+
creds.write_text(CFG["tunnel_credentials"])
|
|
157
|
+
creds.chmod(0o600)
|
|
158
|
+
conf = cf_dir / "config.yml"
|
|
159
|
+
conf.write_text(
|
|
160
|
+
f"tunnel: {CFG['tunnel_id']}\n"
|
|
161
|
+
f"credentials-file: {creds}\n"
|
|
162
|
+
"ingress:\n"
|
|
163
|
+
f" - hostname: {host}\n"
|
|
164
|
+
" service: ssh://localhost:22\n"
|
|
165
|
+
" - service: http_status:404\n"
|
|
166
|
+
)
|
|
167
|
+
cmd = [binary, "--no-autoupdate", "--config", str(conf),
|
|
168
|
+
"tunnel", "run", CFG["tunnel_id"]]
|
|
169
|
+
elif CFG.get("tunnel_token"):
|
|
170
|
+
# Remotely-managed: ingress must already exist in the dashboard, because
|
|
171
|
+
# --token makes cloudflared ignore any local config.
|
|
172
|
+
cmd = [binary, "--no-autoupdate", "tunnel", "run", "--token", CFG["tunnel_token"]]
|
|
173
|
+
else:
|
|
174
|
+
cmd = [binary, "--no-autoupdate", "tunnel", "--url", "ssh://localhost:22"]
|
|
175
|
+
host = None
|
|
176
|
+
|
|
177
|
+
proc = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT)
|
|
178
|
+
if host is None:
|
|
179
|
+
host = wait_for_quick_tunnel_host("/kaggle/working/cloudflared.log")
|
|
180
|
+
return proc, host
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def wait_for_quick_tunnel_host(path, timeout=120):
|
|
184
|
+
"""Quick tunnels only announce their random hostname in their own log."""
|
|
185
|
+
deadline = time.time() + timeout
|
|
186
|
+
while time.time() < deadline:
|
|
187
|
+
text = pathlib.Path(path).read_text() if pathlib.Path(path).exists() else ""
|
|
188
|
+
for token in text.split():
|
|
189
|
+
if token.startswith("https://") and token.endswith(".trycloudflare.com"):
|
|
190
|
+
return token.removeprefix("https://")
|
|
191
|
+
time.sleep(2)
|
|
192
|
+
raise SystemExit("cloudflared never announced a hostname; see cloudflared.log")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def clone_repo():
|
|
196
|
+
remote = CFG.get("git_remote")
|
|
197
|
+
if not remote:
|
|
198
|
+
return None
|
|
199
|
+
if CFG.get("git_deploy_key"):
|
|
200
|
+
key = pathlib.Path("/root/.ssh/git_deploy")
|
|
201
|
+
key.write_text(CFG["git_deploy_key"].rstrip() + "\n")
|
|
202
|
+
key.chmod(0o600)
|
|
203
|
+
add_env("GIT_SSH_COMMAND",
|
|
204
|
+
f"ssh -i {key} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new")
|
|
205
|
+
|
|
206
|
+
dest = pathlib.Path("/kaggle/working") / remote.rstrip("/").split("/")[-1].removesuffix(".git")
|
|
207
|
+
if dest.exists():
|
|
208
|
+
sh(f"git -C {dest} pull --ff-only", check=False)
|
|
209
|
+
else:
|
|
210
|
+
sh(f"git clone {remote} {dest}", check=False)
|
|
211
|
+
return dest
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def snapshot(dest):
|
|
215
|
+
"""Best-effort commit+push so a timeout does not eat the session's work."""
|
|
216
|
+
if not dest or not dest.exists():
|
|
217
|
+
return
|
|
218
|
+
stamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
219
|
+
sh(f"git -C {dest} add -A", check=False)
|
|
220
|
+
sh(f'git -C {dest} -c user.email=kdev@local -c user.name=kdev '
|
|
221
|
+
f'commit -m "kdev autosave {stamp}"', check=False)
|
|
222
|
+
sh(f"git -C {dest} push", check=False)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def end_run():
|
|
226
|
+
"""Make sure nothing below this cell runs once the session is over.
|
|
227
|
+
|
|
228
|
+
Returning normally would let Save & Run All carry on into whatever sits
|
|
229
|
+
under this cell -- after `kdev down`, on your quota, rewriting files that
|
|
230
|
+
were just saved (measured on a real notebook).
|
|
231
|
+
"""
|
|
232
|
+
try:
|
|
233
|
+
ip = get_ipython() # only defined inside a notebook kernel
|
|
234
|
+
except NameError:
|
|
235
|
+
ip = None
|
|
236
|
+
if ip is not None:
|
|
237
|
+
# A notebook: blank every cell after this one. papermill still walks
|
|
238
|
+
# them, finds nothing to run, and the version ends COMPLETE -- where
|
|
239
|
+
# killing the kernel would mark it ERROR.
|
|
240
|
+
ip.input_transformers_cleanup.append(lambda lines: [])
|
|
241
|
+
return
|
|
242
|
+
# A script: nothing after this line may run. Measured to end COMPLETE, with
|
|
243
|
+
# /kaggle/working saved.
|
|
244
|
+
sys.stdout.flush()
|
|
245
|
+
sys.stderr.flush()
|
|
246
|
+
os._exit(0)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
CHECKPOINT_SECONDS = 900
|
|
250
|
+
META_SECONDS = 30
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def main():
|
|
254
|
+
box = install_box()
|
|
255
|
+
sid = session_id()
|
|
256
|
+
# Who started it matters: Kaggle lets only that account cancel it.
|
|
257
|
+
print(f"KDEV_SESSION id={sid} by={CFG.get('run_by', '')}", flush=True)
|
|
258
|
+
# Before anything slow: record which saved versions this session is meant
|
|
259
|
+
# to be built on, so even a session killed during apt-get tells the next
|
|
260
|
+
# one what it was supposed to contain.
|
|
261
|
+
box.init_state(CFG.get("restore_layers") or [], sid, CFG.get("run_by", ""),
|
|
262
|
+
CFG.get("notebook", ""))
|
|
263
|
+
|
|
264
|
+
stage("sshd"); install_sshd(box)
|
|
265
|
+
stage("env"); export_env()
|
|
266
|
+
stage("repo"); repo = clone_repo()
|
|
267
|
+
stage("tunnel"); proc, host = start_tunnel(install_cloudflared())
|
|
268
|
+
|
|
269
|
+
# The CLI greps stdout for this line to know the box is reachable.
|
|
270
|
+
print(f"{READY} host={host} user=root", flush=True)
|
|
271
|
+
print(f"{READY} ssh -o ProxyCommand='cloudflared access ssh --hostname {host}' root@{host}",
|
|
272
|
+
flush=True)
|
|
273
|
+
|
|
274
|
+
deadline = time.time() + CFG["hold_seconds"]
|
|
275
|
+
# So a machine that connects later can say when this box really ends,
|
|
276
|
+
# rather than guess from its own defaults.
|
|
277
|
+
box.update_state(ends=int(deadline))
|
|
278
|
+
stop = {"now": False}
|
|
279
|
+
signal.signal(signal.SIGTERM, lambda *_: stop.__setitem__("now", True))
|
|
280
|
+
next_checkpoint = time.time() + CHECKPOINT_SECONDS
|
|
281
|
+
next_meta = time.time() + META_SECONDS
|
|
282
|
+
try:
|
|
283
|
+
stop_file = pathlib.Path("/kaggle/working/.kdev-stop")
|
|
284
|
+
while time.time() < deadline and not stop["now"]:
|
|
285
|
+
if stop_file.exists():
|
|
286
|
+
print("KDEV_STOP requested", flush=True)
|
|
287
|
+
stop_file.unlink(missing_ok=True)
|
|
288
|
+
break
|
|
289
|
+
if proc.poll() is not None:
|
|
290
|
+
print("cloudflared exited; see cloudflared.log", flush=True)
|
|
291
|
+
break
|
|
292
|
+
# Kaggle commits /kaggle/working only when the run ends, and there
|
|
293
|
+
# is no API to commit mid-run. If the box is lost to an infra fault
|
|
294
|
+
# at hour 7 of 9, the teardown push never happens -- so push on a
|
|
295
|
+
# timer too, when there is a remote to push to.
|
|
296
|
+
if repo and time.time() >= next_checkpoint:
|
|
297
|
+
print("KDEV_CHECKPOINT", flush=True)
|
|
298
|
+
snapshot(repo)
|
|
299
|
+
next_checkpoint = time.time() + CHECKPOINT_SECONDS
|
|
300
|
+
# Kaggle's snapshot keeps file contents but drops symlinks, empty
|
|
301
|
+
# dirs and exec bits. Note them often: a kill costs at most one
|
|
302
|
+
# interval of metadata, never the files.
|
|
303
|
+
if time.time() >= next_meta:
|
|
304
|
+
try:
|
|
305
|
+
box.record_meta()
|
|
306
|
+
except Exception as e: # a bad scan must never end the session
|
|
307
|
+
print(f"KDEV_META failed: {e}", flush=True)
|
|
308
|
+
next_meta = time.time() + META_SECONDS
|
|
309
|
+
# A heartbeat keeps the log stream flowing so the CLI can tell the
|
|
310
|
+
# difference between 'alive and idle' and 'dead'.
|
|
311
|
+
left = int(deadline - time.time())
|
|
312
|
+
print(f"KDEV_ALIVE seconds_left={left}", flush=True)
|
|
313
|
+
# Short sleeps so a stop request is noticed promptly.
|
|
314
|
+
for _ in range(6):
|
|
315
|
+
if stop_file.exists() or stop["now"]:
|
|
316
|
+
break
|
|
317
|
+
time.sleep(5)
|
|
318
|
+
finally:
|
|
319
|
+
try:
|
|
320
|
+
box.record_meta()
|
|
321
|
+
except Exception as e:
|
|
322
|
+
print(f"KDEV_META failed: {e}", flush=True)
|
|
323
|
+
snapshot(repo)
|
|
324
|
+
proc.terminate()
|
|
325
|
+
print("KDEV_DONE", flush=True)
|
|
326
|
+
end_run()
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
main()
|
|
330
|
+
'''
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def render(
|
|
334
|
+
*,
|
|
335
|
+
ssh_public_key: str,
|
|
336
|
+
tunnel_hostname: str,
|
|
337
|
+
hold_seconds: int,
|
|
338
|
+
run_by: str = "",
|
|
339
|
+
tunnel_token: str = "",
|
|
340
|
+
tunnel_credentials: str = "",
|
|
341
|
+
tunnel_id: str = "",
|
|
342
|
+
git_remote: str = "",
|
|
343
|
+
git_deploy_key: str = "",
|
|
344
|
+
restore_layers: list[str] | None = None,
|
|
345
|
+
notebook: str = "",
|
|
346
|
+
) -> str:
|
|
347
|
+
if tunnel_credentials and not (tunnel_id and tunnel_hostname):
|
|
348
|
+
raise ValueError("tunnel credentials need both tunnel_id and tunnel_hostname")
|
|
349
|
+
cfg = {
|
|
350
|
+
"ssh_public_key": ssh_public_key,
|
|
351
|
+
"tunnel_token": tunnel_token,
|
|
352
|
+
"tunnel_credentials": tunnel_credentials,
|
|
353
|
+
"tunnel_id": tunnel_id,
|
|
354
|
+
"tunnel_hostname": tunnel_hostname,
|
|
355
|
+
"hold_seconds": hold_seconds,
|
|
356
|
+
"run_by": run_by,
|
|
357
|
+
"git_remote": git_remote,
|
|
358
|
+
"git_deploy_key": git_deploy_key,
|
|
359
|
+
"restore_layers": list(restore_layers or []),
|
|
360
|
+
"notebook": notebook,
|
|
361
|
+
"box_script": BOX_SCRIPT.read_text(),
|
|
362
|
+
}
|
|
363
|
+
# json.dumps escapes every quote and backslash, so the blob can never close
|
|
364
|
+
# the r"""...""" literal it lands in. Parsing the result proves it rather
|
|
365
|
+
# than assuming it -- a broken render fails here, not 3 minutes into a
|
|
366
|
+
# Kaggle session.
|
|
367
|
+
blob = json.dumps(cfg)
|
|
368
|
+
src = (
|
|
369
|
+
_TEMPLATE.replace("__CONFIG_JSON__", blob)
|
|
370
|
+
.replace("__READY_MARKER__", READY_MARKER)
|
|
371
|
+
.replace("__STAGE_MARKER__", STAGE_MARKER)
|
|
372
|
+
)
|
|
373
|
+
try:
|
|
374
|
+
ast.parse(src)
|
|
375
|
+
except SyntaxError as e:
|
|
376
|
+
raise ValueError(f"config broke the generated script: {e}") from e
|
|
377
|
+
return src
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
BOX_SCRIPT = Path(__file__).with_name("kdev_box.py")
|
|
381
|
+
|
|
382
|
+
#: Settings a second machine can take from the notebook itself, so joining an
|
|
383
|
+
#: existing workspace does not mean setting the tunnel up all over again. The
|
|
384
|
+
#: SSH key is deliberately absent: every machine brings its own.
|
|
385
|
+
SHARED_SETTINGS = (
|
|
386
|
+
"tunnel_hostname",
|
|
387
|
+
"tunnel_id",
|
|
388
|
+
"tunnel_credentials",
|
|
389
|
+
"tunnel_token",
|
|
390
|
+
"git_remote",
|
|
391
|
+
"git_deploy_key",
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
_CFG_RE = re.compile(r'CFG = json\.loads\(r"""(.*?)"""\)', re.DOTALL)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _source_text(source: str) -> str:
|
|
398
|
+
"""The code in a notebook or script, as one string."""
|
|
399
|
+
stripped = (source or "").strip()
|
|
400
|
+
if stripped.startswith("{"):
|
|
401
|
+
try:
|
|
402
|
+
nb = json.loads(stripped)
|
|
403
|
+
except json.JSONDecodeError:
|
|
404
|
+
return source or ""
|
|
405
|
+
if isinstance(nb, dict) and isinstance(nb.get("cells"), list):
|
|
406
|
+
return "\n".join(
|
|
407
|
+
"".join(c.get("source") or "")
|
|
408
|
+
if isinstance(c.get("source"), list)
|
|
409
|
+
else str(c.get("source") or "")
|
|
410
|
+
for c in nb["cells"]
|
|
411
|
+
)
|
|
412
|
+
return source or ""
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def extract_config(source: str) -> dict:
|
|
416
|
+
"""The kdev config baked into a notebook's source, or {} if there is none."""
|
|
417
|
+
m = _CFG_RE.search(_source_text(source))
|
|
418
|
+
if not m:
|
|
419
|
+
return {}
|
|
420
|
+
try:
|
|
421
|
+
cfg = json.loads(m.group(1))
|
|
422
|
+
except json.JSONDecodeError:
|
|
423
|
+
return {}
|
|
424
|
+
return {k: cfg[k] for k in SHARED_SETTINGS if cfg.get(k)} if isinstance(cfg, dict) else {}
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
CELL_HEADER = f"{MARKER} -- this cell is the session: it ends the run when it stops, so nothing below it ever executes."
|
|
428
|
+
SCRIPT_BEGIN = f"{MARKER} >>>"
|
|
429
|
+
SCRIPT_END = f"{MARKER} <<<"
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
#: What Kaggle's papermill runner needs to start a notebook at all. Without a
|
|
433
|
+
#: kernelspec it fails before the first cell: "No kernel name found in
|
|
434
|
+
#: notebook" (measured).
|
|
435
|
+
KERNELSPEC = {"name": "python3", "display_name": "Python 3", "language": "python"}
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _runnable(nb: dict) -> dict:
|
|
439
|
+
meta = nb.setdefault("metadata", {})
|
|
440
|
+
if not (meta.get("kernelspec") or {}).get("name"):
|
|
441
|
+
meta["kernelspec"] = dict(KERNELSPEC)
|
|
442
|
+
meta.setdefault("language_info", {"name": "python"})
|
|
443
|
+
nb.setdefault("nbformat", 4)
|
|
444
|
+
nb.setdefault("nbformat_minor", 4)
|
|
445
|
+
return nb
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _is_kdev_cell(cell: dict) -> bool:
|
|
449
|
+
src = cell.get("source") or ""
|
|
450
|
+
text = "".join(src) if isinstance(src, list) else str(src)
|
|
451
|
+
return MARKER in text.split("\n", 3)[0] if text else False
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def inject(existing: str, script: str) -> tuple[str, str]:
|
|
455
|
+
"""Put the bootstrap in front of a notebook without destroying it.
|
|
456
|
+
|
|
457
|
+
Save & Run All executes top to bottom, and the injected cell blocks for the
|
|
458
|
+
whole session, so the user's own cells never run -- but they are still
|
|
459
|
+
there, and still theirs. Re-running replaces kdev's cell rather than
|
|
460
|
+
stacking another one on top.
|
|
461
|
+
|
|
462
|
+
Returns (source, kernel_type).
|
|
463
|
+
"""
|
|
464
|
+
stripped = (existing or "").strip()
|
|
465
|
+
|
|
466
|
+
if stripped.startswith("{"):
|
|
467
|
+
try:
|
|
468
|
+
nb = json.loads(stripped)
|
|
469
|
+
except json.JSONDecodeError:
|
|
470
|
+
nb = None
|
|
471
|
+
if isinstance(nb, dict) and isinstance(nb.get("cells"), list):
|
|
472
|
+
cells = [c for c in nb["cells"] if not _is_kdev_cell(c)]
|
|
473
|
+
nb["cells"] = [
|
|
474
|
+
{
|
|
475
|
+
"cell_type": "code",
|
|
476
|
+
"execution_count": None,
|
|
477
|
+
"metadata": {},
|
|
478
|
+
"outputs": [],
|
|
479
|
+
"source": (CELL_HEADER + "\n" + script).splitlines(keepends=True),
|
|
480
|
+
},
|
|
481
|
+
*cells,
|
|
482
|
+
]
|
|
483
|
+
return json.dumps(_runnable(nb)), "notebook"
|
|
484
|
+
|
|
485
|
+
if stripped:
|
|
486
|
+
# A plain script: keep the user's code below a delimited kdev block.
|
|
487
|
+
body = stripped
|
|
488
|
+
if SCRIPT_BEGIN in body and SCRIPT_END in body:
|
|
489
|
+
head, _, rest = body.partition(SCRIPT_BEGIN)
|
|
490
|
+
_, _, tail = rest.partition(SCRIPT_END)
|
|
491
|
+
body = (head + tail).strip()
|
|
492
|
+
block = f"{SCRIPT_BEGIN}\n{script}\n{SCRIPT_END}"
|
|
493
|
+
return (block + ("\n\n" + body if body else "") + "\n"), "script"
|
|
494
|
+
|
|
495
|
+
# Nothing there yet: a one-cell notebook, so cells can be added underneath.
|
|
496
|
+
return json.dumps(
|
|
497
|
+
_runnable(
|
|
498
|
+
{
|
|
499
|
+
"cells": [
|
|
500
|
+
{
|
|
501
|
+
"cell_type": "code",
|
|
502
|
+
"execution_count": None,
|
|
503
|
+
"metadata": {},
|
|
504
|
+
"outputs": [],
|
|
505
|
+
"source": (CELL_HEADER + "\n" + script).splitlines(keepends=True),
|
|
506
|
+
}
|
|
507
|
+
],
|
|
508
|
+
}
|
|
509
|
+
)
|
|
510
|
+
), "notebook"
|
kdev/cli.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""kdev: a Kaggle notebook as your remote dev box.
|
|
2
|
+
|
|
3
|
+
Start it, connect from VS Code or a shell, and leave whenever you like; your
|
|
4
|
+
files come back on the next start, from any account in your group, on any
|
|
5
|
+
machine.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
import typer
|
|
14
|
+
import typer.main
|
|
15
|
+
|
|
16
|
+
from . import __version__, api, auth, tunnel, ui
|
|
17
|
+
from .commands import account, box, home, settings, workspace
|
|
18
|
+
from .commands import doctor as doctor_cmd
|
|
19
|
+
from .commands import tunnel as tunnel_cmd
|
|
20
|
+
from .errors import Cancelled, KdevError
|
|
21
|
+
|
|
22
|
+
app = typer.Typer(
|
|
23
|
+
name="kdev",
|
|
24
|
+
help=__doc__,
|
|
25
|
+
no_args_is_help=False,
|
|
26
|
+
rich_markup_mode="rich",
|
|
27
|
+
context_settings={"help_option_names": ["-h", "--help"]},
|
|
28
|
+
# Typer's default crash report prints every local variable -- which here
|
|
29
|
+
# includes OAuth tokens and tunnel credentials. Never.
|
|
30
|
+
pretty_exceptions_show_locals=False,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _version(value: bool) -> None:
|
|
35
|
+
if value:
|
|
36
|
+
print(f"kdev {__version__}")
|
|
37
|
+
raise typer.Exit()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@app.callback(invoke_without_command=True)
|
|
41
|
+
def root(
|
|
42
|
+
ctx: typer.Context,
|
|
43
|
+
version: bool = typer.Option(
|
|
44
|
+
None,
|
|
45
|
+
"--version",
|
|
46
|
+
"-V",
|
|
47
|
+
callback=_version,
|
|
48
|
+
is_eager=True,
|
|
49
|
+
help="Print the version and exit.",
|
|
50
|
+
),
|
|
51
|
+
verbose: bool = typer.Option(
|
|
52
|
+
False, "--verbose", "-v", help="Log every Kaggle API call to stderr."
|
|
53
|
+
),
|
|
54
|
+
) -> None:
|
|
55
|
+
"""With no command: the overview, and what to do next."""
|
|
56
|
+
if verbose:
|
|
57
|
+
logging.basicConfig(
|
|
58
|
+
level=logging.DEBUG, stream=sys.stderr, format=" %(name)s %(message)s"
|
|
59
|
+
)
|
|
60
|
+
# Our own calls only; httpx's per-request chatter is noise here.
|
|
61
|
+
for noisy in ("httpx", "httpcore", "asyncio"):
|
|
62
|
+
logging.getLogger(noisy).setLevel(logging.WARNING)
|
|
63
|
+
if ctx.invoked_subcommand is None:
|
|
64
|
+
home.overview(_dispatch)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# The box -------------------------------------------------------------------
|
|
68
|
+
app.command("up")(box.up)
|
|
69
|
+
app.command("down")(box.down)
|
|
70
|
+
app.command("ssh", context_settings={"allow_extra_args": True, "ignore_unknown_options": True})(
|
|
71
|
+
box.ssh
|
|
72
|
+
)
|
|
73
|
+
app.command("status")(box.status)
|
|
74
|
+
app.command("logs")(box.logs)
|
|
75
|
+
app.command("restore")(box.restore_cmd)
|
|
76
|
+
app.command("backup")(box.backup)
|
|
77
|
+
# Accounts, workspace, settings, tunnel ---------------------------------------
|
|
78
|
+
app.add_typer(account.app, name="account")
|
|
79
|
+
app.add_typer(workspace.app, name="workspace")
|
|
80
|
+
app.add_typer(settings.app, name="config")
|
|
81
|
+
app.add_typer(tunnel_cmd.app, name="tunnel")
|
|
82
|
+
app.command("doctor")(doctor_cmd.doctor)
|
|
83
|
+
app.command("setup")(home.setup)
|
|
84
|
+
# Muscle memory, not advertised ------------------------------------------------
|
|
85
|
+
app.command("ps", hidden=True)(box.status)
|
|
86
|
+
app.command("login", hidden=True)(account.add)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _dispatch(argv: list[str]) -> None:
|
|
90
|
+
"""Run a command picked from the overview, as if it had been typed --
|
|
91
|
+
including its exit status, which a non-standalone run returns rather than
|
|
92
|
+
raises. Re-raised so it unwinds through `main` like a typed command's."""
|
|
93
|
+
code = typer.main.get_command(app).main(args=argv, prog_name="kdev", standalone_mode=False)
|
|
94
|
+
if isinstance(code, int) and code:
|
|
95
|
+
raise typer.Exit(code)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _explain(e: api.KaggleError) -> tuple[str, str]:
|
|
99
|
+
text = str(e)
|
|
100
|
+
if e.status == 401:
|
|
101
|
+
return text, "Sign that account in again: kdev account add <name>"
|
|
102
|
+
if e.status == 403:
|
|
103
|
+
return text, "That account is not allowed to do this. Is it in the group? kdev workspace"
|
|
104
|
+
if e.status == 404:
|
|
105
|
+
return text, "Kaggle cannot find it. Check the workspace: kdev workspace"
|
|
106
|
+
if e.status == 429:
|
|
107
|
+
return text, "Kaggle is rate-limiting; wait a minute and retry."
|
|
108
|
+
return text, "Run it again with -v to see each API call."
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def main() -> None:
|
|
112
|
+
verbose = any(a in ("-v", "--verbose") for a in sys.argv[1:3])
|
|
113
|
+
try:
|
|
114
|
+
# Not standalone, so errors reach the handlers below -- which means a
|
|
115
|
+
# command's `typer.Exit(n)` comes back as a return value, not an exit.
|
|
116
|
+
# Pass it on, or every non-zero status (doctor, ssh, restore) is lost.
|
|
117
|
+
code = app(prog_name="kdev", standalone_mode=False)
|
|
118
|
+
if isinstance(code, int) and code:
|
|
119
|
+
sys.exit(code)
|
|
120
|
+
except Cancelled:
|
|
121
|
+
ui.errconsole.print(ui.Text(" cancelled", style="kdev.muted"))
|
|
122
|
+
sys.exit(130)
|
|
123
|
+
except KdevError as e:
|
|
124
|
+
ui.err(e.message, e.hint)
|
|
125
|
+
sys.exit(1)
|
|
126
|
+
except api.KaggleError as e:
|
|
127
|
+
ui.err(*_explain(e))
|
|
128
|
+
sys.exit(1)
|
|
129
|
+
except (auth.AuthError, tunnel.TunnelError) as e:
|
|
130
|
+
ui.err(str(e))
|
|
131
|
+
sys.exit(1)
|
|
132
|
+
except KeyboardInterrupt:
|
|
133
|
+
ui.errconsole.print(ui.Text("\n interrupted", style="kdev.muted"))
|
|
134
|
+
sys.exit(130)
|
|
135
|
+
except typer.Abort:
|
|
136
|
+
ui.errconsole.print(ui.Text("\n interrupted", style="kdev.muted"))
|
|
137
|
+
sys.exit(130)
|
|
138
|
+
except typer.TyperException as e:
|
|
139
|
+
# Usage mistakes -- a bad option, a missing argument -- in the
|
|
140
|
+
# parser's own words, which are good, plus where to look.
|
|
141
|
+
show = getattr(e, "show", None)
|
|
142
|
+
if callable(show):
|
|
143
|
+
show()
|
|
144
|
+
else:
|
|
145
|
+
ui.err(str(e), "kdev --help lists every command and option.")
|
|
146
|
+
sys.exit(getattr(e, "exit_code", 2))
|
|
147
|
+
except typer.Exit as e:
|
|
148
|
+
sys.exit(e.exit_code)
|
|
149
|
+
except SystemExit:
|
|
150
|
+
raise
|
|
151
|
+
except Exception as e: # the last line of defence: never a raw traceback
|
|
152
|
+
if verbose:
|
|
153
|
+
raise
|
|
154
|
+
ui.err(
|
|
155
|
+
f"kdev hit an unexpected error: {type(e).__name__}: {e}",
|
|
156
|
+
"Run it again with -v for the full traceback, and report it.",
|
|
157
|
+
)
|
|
158
|
+
sys.exit(1)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
if __name__ == "__main__":
|
|
162
|
+
main()
|