kdev-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kdev/__init__.py +7 -0
- kdev/__main__.py +3 -0
- kdev/api.py +408 -0
- kdev/auth.py +115 -0
- kdev/bootstrap.py +510 -0
- kdev/cli.py +162 -0
- kdev/cloudflared.py +179 -0
- kdev/commands/__init__.py +5 -0
- kdev/commands/_common.py +51 -0
- kdev/commands/account.py +148 -0
- kdev/commands/box.py +694 -0
- kdev/commands/doctor.py +127 -0
- kdev/commands/home.py +92 -0
- kdev/commands/settings.py +164 -0
- kdev/commands/tunnel.py +78 -0
- kdev/commands/workspace.py +210 -0
- kdev/config.py +181 -0
- kdev/errors.py +25 -0
- kdev/kdev_box.py +538 -0
- kdev/notebook.py +261 -0
- kdev/persistence.py +314 -0
- kdev/quota.py +102 -0
- kdev/safety.py +72 -0
- kdev/session.py +284 -0
- kdev/sshcfg.py +72 -0
- kdev/tunnel.py +259 -0
- kdev/ui.py +666 -0
- kdev/wizard.py +251 -0
- kdev_cli-0.1.3.dist-info/METADATA +598 -0
- kdev_cli-0.1.3.dist-info/RECORD +33 -0
- kdev_cli-0.1.3.dist-info/WHEEL +4 -0
- kdev_cli-0.1.3.dist-info/entry_points.txt +2 -0
- kdev_cli-0.1.3.dist-info/licenses/LICENSE +21 -0
kdev/safety.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Guard against overwriting a notebook someone is actually using.
|
|
2
|
+
|
|
3
|
+
`SaveKernel` replaces a notebook's entire source. kdev generates its own
|
|
4
|
+
bootstrap script, so pointing it at a notebook that holds real work destroys
|
|
5
|
+
that work -- and an uncommitted editor draft has no version history to restore
|
|
6
|
+
from. Nothing may be overwritten unless kdev wrote it, or the user says so
|
|
7
|
+
having been told exactly what is at stake.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import api, bootstrap
|
|
16
|
+
from . import config as _config
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def backups_dir() -> Path:
|
|
20
|
+
return _config.CONFIG_DIR / "backups"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def is_kdev_notebook(source: str) -> bool:
|
|
24
|
+
return bootstrap.MARKER in source
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def is_empty(source: str) -> bool:
|
|
28
|
+
"""Blank, or the placeholder `kdev workspace create` writes."""
|
|
29
|
+
stripped = source.strip()
|
|
30
|
+
return not stripped or "source is generated per session" in stripped
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
#: Every `kdev up` replaces the source, so every `kdev up` writes a backup.
|
|
34
|
+
#: Without a cap that directory grows for as long as the tool is used.
|
|
35
|
+
KEEP_BACKUPS = 20
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def back_up(slug: str, source: str) -> Path | None:
|
|
39
|
+
"""Keep a local copy of whatever we are about to replace."""
|
|
40
|
+
if not source.strip():
|
|
41
|
+
return None
|
|
42
|
+
prefix = slug.replace("/", "_")
|
|
43
|
+
path = backups_dir() / f"{prefix}-{time.strftime('%Y%m%d-%H%M%S')}.txt"
|
|
44
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
path.write_text(source)
|
|
46
|
+
prune(prefix)
|
|
47
|
+
return path
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def prune(prefix: str, keep: int = KEEP_BACKUPS) -> None:
|
|
51
|
+
"""Keep the newest `keep` backups for one notebook, drop the rest."""
|
|
52
|
+
old = sorted(backups_dir().glob(f"{prefix}-*.txt"), key=lambda p: p.name, reverse=True)[keep:]
|
|
53
|
+
for path in old:
|
|
54
|
+
path.unlink(missing_ok=True)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def check(creds: api.Creds, slug: str) -> tuple[str, str, Path | None]:
|
|
58
|
+
"""Classify a notebook before kdev writes to it.
|
|
59
|
+
|
|
60
|
+
Returns (verdict, source, backup_path) where verdict is "safe" when kdev
|
|
61
|
+
owns the notebook or it is empty, and "occupied" when it holds other code.
|
|
62
|
+
"""
|
|
63
|
+
try:
|
|
64
|
+
source = api.kernel_source(creds, slug)
|
|
65
|
+
except api.KaggleError:
|
|
66
|
+
# A notebook that does not exist yet is safe to create.
|
|
67
|
+
return "safe", "", None
|
|
68
|
+
|
|
69
|
+
backup = back_up(slug, source)
|
|
70
|
+
if is_empty(source) or is_kdev_notebook(source):
|
|
71
|
+
return "safe", source, backup
|
|
72
|
+
return "occupied", source, backup
|
kdev/session.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""A live session: finding it, reaching it, restoring into it, stopping it.
|
|
2
|
+
|
|
3
|
+
Pure orchestration over the Kaggle API, ssh and the on-box restore; the
|
|
4
|
+
commands in `kdev.commands` decide what to show around it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
import shutil
|
|
11
|
+
import subprocess
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
|
|
15
|
+
from . import api, bootstrap, cloudflared, config, persistence, sshcfg, ui
|
|
16
|
+
from .errors import KdevError
|
|
17
|
+
|
|
18
|
+
#: A session that never reports a tunnel is using quota for nothing, so the
|
|
19
|
+
#: wait is bounded. Kaggle's own queue can be slow, hence the generous default.
|
|
20
|
+
READY_TIMEOUT = 1800
|
|
21
|
+
|
|
22
|
+
_READY = re.compile(rf"{bootstrap.READY_MARKER} host=(\S+)")
|
|
23
|
+
_STAGE = re.compile(rf"{bootstrap.STAGE_MARKER} (\w+)")
|
|
24
|
+
_SESSION = re.compile(r"KDEV_SESSION id=(\d+)(?: by=(\S*))?")
|
|
25
|
+
|
|
26
|
+
EDITORS = ("code", "cursor", "code-insiders", "windsurf")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def ssh_ready(cfg: config.Config) -> None:
|
|
30
|
+
"""Make `ssh <alias>` -- and VS Code's host of that name -- work here.
|
|
31
|
+
|
|
32
|
+
A named tunnel's hostname never changes, so the block needs no box to look
|
|
33
|
+
at: a machine that did not start the running box, or that ran `kdev down`
|
|
34
|
+
on the last one, can still reach the next.
|
|
35
|
+
"""
|
|
36
|
+
if cfg.tunnel_hostname and cfg.ssh_host_alias and not sshcfg.has_block():
|
|
37
|
+
sshcfg.write(cfg.ssh_host_alias, cfg.tunnel_hostname, cloudflared_path=cloudflared.find())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def box_gone(cfg: config.Config) -> None:
|
|
41
|
+
"""After a box stops. Only a quick tunnel's block is stale now: its
|
|
42
|
+
hostname dies with the box. A named one is where the next box will be."""
|
|
43
|
+
if not cfg.tunnel_hostname:
|
|
44
|
+
sshcfg.clear()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def reachable(alias: str, timeout: int = 10) -> bool:
|
|
48
|
+
"""Can ssh open a channel to the box right now?"""
|
|
49
|
+
if not alias:
|
|
50
|
+
return False
|
|
51
|
+
try:
|
|
52
|
+
r = subprocess.run(
|
|
53
|
+
["ssh", "-T", "-o", "BatchMode=yes", "-o", f"ConnectTimeout={timeout}", alias, "true"],
|
|
54
|
+
capture_output=True,
|
|
55
|
+
timeout=timeout + 15,
|
|
56
|
+
)
|
|
57
|
+
except (subprocess.SubprocessError, OSError):
|
|
58
|
+
return False
|
|
59
|
+
return r.returncode == 0
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def await_ready(
|
|
63
|
+
creds: api.Creds,
|
|
64
|
+
slug: str,
|
|
65
|
+
board: ui.Stages | None = None,
|
|
66
|
+
timeout: int = READY_TIMEOUT,
|
|
67
|
+
seen: dict | None = None,
|
|
68
|
+
) -> str | None:
|
|
69
|
+
"""Follow the session log until the box announces its tunnel.
|
|
70
|
+
|
|
71
|
+
Drives `board` from the stage markers, and records the session id and who
|
|
72
|
+
started it into `seen` as they go past.
|
|
73
|
+
"""
|
|
74
|
+
deadline = time.time() + timeout
|
|
75
|
+
for line in api.stream_logs(creds, slug):
|
|
76
|
+
if time.time() > deadline:
|
|
77
|
+
if board:
|
|
78
|
+
board.note(f"gave up after {timeout // 60} min without a tunnel")
|
|
79
|
+
return None
|
|
80
|
+
if not line.strip():
|
|
81
|
+
continue
|
|
82
|
+
found = _SESSION.search(line)
|
|
83
|
+
if found and seen is not None:
|
|
84
|
+
seen["session"], seen["by"] = found.group(1), found.group(2) or ""
|
|
85
|
+
if board:
|
|
86
|
+
stage = _STAGE.search(line)
|
|
87
|
+
if stage:
|
|
88
|
+
board.advance(stage.group(1))
|
|
89
|
+
else:
|
|
90
|
+
board.note(line.strip())
|
|
91
|
+
ready = _READY.search(line)
|
|
92
|
+
if ready:
|
|
93
|
+
return ready.group(1)
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
#: How long a stopped box may take to save. Kaggle uploads /kaggle/working
|
|
98
|
+
#: before it calls the run finished: measured ~110 s for 1 GB.
|
|
99
|
+
SAVE_TIMEOUT = 1800
|
|
100
|
+
|
|
101
|
+
#: find_live_box's answer for a box that was told to stop and is saving.
|
|
102
|
+
STOPPING = "stopping"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def wait_saved(
|
|
106
|
+
creds: api.Creds, slug: str, running_too: bool = False, timeout: int = SAVE_TIMEOUT
|
|
107
|
+
) -> None:
|
|
108
|
+
"""Block while a session that stopped is still saving its files.
|
|
109
|
+
|
|
110
|
+
Planning a restore before then builds on a partial or older version, and
|
|
111
|
+
everything done in the session that stopped is dropped from then on. A
|
|
112
|
+
cancelled box saves as CANCEL_REQUESTED; one told to stop saves while
|
|
113
|
+
still RUNNING, which is `running_too`.
|
|
114
|
+
"""
|
|
115
|
+
busy = {api.SAVING, *(api.LIVE_STATES if running_too else ())}
|
|
116
|
+
deadline = time.time() + timeout
|
|
117
|
+
while api.session_status(creds, slug).get("status") in busy:
|
|
118
|
+
if time.time() > deadline:
|
|
119
|
+
raise KdevError(
|
|
120
|
+
"The last session is still saving its files.",
|
|
121
|
+
"Starting now would leave them out. Try again in a few minutes.",
|
|
122
|
+
)
|
|
123
|
+
time.sleep(3)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def stopping(creds: api.Creds, slug: str, window: float = 20) -> bool:
|
|
127
|
+
"""Whether the running box was already told to stop, and is saving.
|
|
128
|
+
|
|
129
|
+
After `kdev down` Kaggle keeps the run RUNNING for as long as the upload
|
|
130
|
+
takes -- minutes, for a few GB -- with the tunnel already gone: from
|
|
131
|
+
outside, just like a live box that cannot be reached. Its log says which.
|
|
132
|
+
"""
|
|
133
|
+
deadline = time.time() + window
|
|
134
|
+
try:
|
|
135
|
+
# A box told to stop logs it within 5 s; `idle` outlasts that.
|
|
136
|
+
for line in api.stream_logs(creds, slug, wait_seconds=30, idle=8):
|
|
137
|
+
if "KDEV_STOP" in line or "KDEV_DONE" in line:
|
|
138
|
+
return True
|
|
139
|
+
if time.time() > deadline:
|
|
140
|
+
break
|
|
141
|
+
except api.KaggleError:
|
|
142
|
+
pass
|
|
143
|
+
return False
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def find_live_box(cfg: config.Config, creds: api.Creds, target: str) -> str:
|
|
147
|
+
"""Hostname of a kdev box running on `target`, STOPPING if it is on its
|
|
148
|
+
way out, or "" if there is none.
|
|
149
|
+
|
|
150
|
+
The session status alone is not enough: a Quick Save reads as running for a
|
|
151
|
+
few seconds, and so does a session started in the Kaggle editor. A kdev box
|
|
152
|
+
is one that announced its tunnel -- and one still booting will do so
|
|
153
|
+
shortly, so follow its log rather than guess.
|
|
154
|
+
"""
|
|
155
|
+
if cfg.tunnel_hostname and reachable(cfg.ssh_host_alias):
|
|
156
|
+
return cfg.tunnel_hostname
|
|
157
|
+
if stopping(creds, target):
|
|
158
|
+
return STOPPING
|
|
159
|
+
return await_ready(creds, target, timeout=900) or ""
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def live_session_id(creds: api.Creds, slug: str, timeout: int = 60) -> tuple[str, str]:
|
|
163
|
+
"""(session id, username that started it), from the box's boot line."""
|
|
164
|
+
deadline = time.time() + timeout
|
|
165
|
+
try:
|
|
166
|
+
for line in api.stream_logs(creds, slug, wait_seconds=30):
|
|
167
|
+
found = _SESSION.search(line)
|
|
168
|
+
if found:
|
|
169
|
+
return found.group(1), found.group(2) or ""
|
|
170
|
+
if bootstrap.READY_MARKER in line or time.time() > deadline:
|
|
171
|
+
break
|
|
172
|
+
except api.KaggleError:
|
|
173
|
+
pass
|
|
174
|
+
return "", ""
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def cancel_as_owner(cfg: config.Config, sid: int, runner: str = "") -> str:
|
|
178
|
+
"""Cancel a session as the account that started it. Returns that account.
|
|
179
|
+
|
|
180
|
+
Kaggle refuses a cancel from anyone else -- measured: 403
|
|
181
|
+
`kernelSessions.cancel` even for an Editor of the notebook. So try the
|
|
182
|
+
known starter first, then every signed-in account (a box that predates
|
|
183
|
+
the `by=` field has no name to go on).
|
|
184
|
+
"""
|
|
185
|
+
names = sorted(cfg.profiles, key=lambda n: cfg.profiles[n].username != runner)
|
|
186
|
+
for name in names:
|
|
187
|
+
try:
|
|
188
|
+
api.cancel_session(cfg.profile(name).creds, sid)
|
|
189
|
+
except api.KaggleError:
|
|
190
|
+
continue
|
|
191
|
+
return name
|
|
192
|
+
return ""
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def restore(
|
|
196
|
+
creds: api.Creds,
|
|
197
|
+
notebook: str,
|
|
198
|
+
alias: str,
|
|
199
|
+
layers: list[str],
|
|
200
|
+
note: Callable[[str], None] | None = None,
|
|
201
|
+
) -> dict:
|
|
202
|
+
"""Resolve fresh URLs for `layers` and have the box restore itself.
|
|
203
|
+
|
|
204
|
+
Never raises for a restore that did not finish: the box is up either way,
|
|
205
|
+
and the unfinished state is recorded on it for `kdev restore` or the next
|
|
206
|
+
`kdev up` to complete.
|
|
207
|
+
"""
|
|
208
|
+
say = note or (lambda _text: None)
|
|
209
|
+
say("resolving saved files…")
|
|
210
|
+
plan = persistence.build_plan(creds, notebook, layers)
|
|
211
|
+
if not sum(len(layer["files"]) for layer in plan["layers"]):
|
|
212
|
+
say("saved version is empty")
|
|
213
|
+
return {"restored": True, "done": 0, "total": 0}
|
|
214
|
+
|
|
215
|
+
def progress(state: dict) -> None:
|
|
216
|
+
if state.get("total"):
|
|
217
|
+
say(f"{state.get('done', 0)}/{state['total']} files")
|
|
218
|
+
|
|
219
|
+
ok, last = persistence.restore_on_box(alias, plan, progress)
|
|
220
|
+
if not ok:
|
|
221
|
+
say(last.get("error") or "restore did not finish")
|
|
222
|
+
return {**last, "restored": ok}
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def report_restore(state: dict) -> None:
|
|
226
|
+
if not state:
|
|
227
|
+
return
|
|
228
|
+
if not state.get("restored"):
|
|
229
|
+
ui.warn(
|
|
230
|
+
"Not all your files are back yet: "
|
|
231
|
+
+ (state.get("error") or "the restore did not finish"),
|
|
232
|
+
"Finish it with `kdev restore`; it never overwrites work from this session.",
|
|
233
|
+
)
|
|
234
|
+
return
|
|
235
|
+
if state.get("total"):
|
|
236
|
+
ui.ok(f"restored {state.get('done', 0)} file(s) into /kaggle/working")
|
|
237
|
+
elif "total" in state:
|
|
238
|
+
ui.ok("nothing missing: every saved file is already on the box")
|
|
239
|
+
missing = state.get("missing") or []
|
|
240
|
+
for name in missing[:10]:
|
|
241
|
+
ui.warn(f"could not restore {name}")
|
|
242
|
+
if len(missing) > 10:
|
|
243
|
+
ui.warn(f"… and {len(missing) - 10} more")
|
|
244
|
+
if missing:
|
|
245
|
+
ui.hint("Retry with `kdev restore`.")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def describe_running(
|
|
249
|
+
cfg: config.Config, creds: api.Creds, target: str, hours: float, gpu: str, account: str
|
|
250
|
+
) -> tuple[float, str, str]:
|
|
251
|
+
"""(hours left, accelerator, started by) for a box someone else may have
|
|
252
|
+
started: its own record, not this machine's defaults."""
|
|
253
|
+
state = persistence.box_state(cfg.ssh_host_alias)
|
|
254
|
+
if state.get("ends"):
|
|
255
|
+
hours = max(0.0, (float(state["ends"]) - time.time()) / 3600)
|
|
256
|
+
account = state.get("run_by") or account
|
|
257
|
+
try:
|
|
258
|
+
shape = api.get_kernel(creds, target).get("machineShape") or ""
|
|
259
|
+
gpu = next((k for k, v in api.SHAPES.items() if v == shape), "none")
|
|
260
|
+
except api.KaggleError:
|
|
261
|
+
pass
|
|
262
|
+
return round(hours, 1), ui.gpu_label(gpu), account
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def editor() -> str:
|
|
266
|
+
"""The first VS Code-compatible CLI on PATH, or ""."""
|
|
267
|
+
return next((exe for exe in EDITORS if shutil.which(exe)), "")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def open_editor(alias: str, remote_path: str = "/kaggle/working") -> bool:
|
|
271
|
+
exe = editor()
|
|
272
|
+
uri = f"vscode-remote://ssh-remote+{alias}{remote_path}"
|
|
273
|
+
if not exe:
|
|
274
|
+
ui.hint(f"No VS Code CLI on PATH. Open this in VS Code: {uri}")
|
|
275
|
+
return False
|
|
276
|
+
subprocess.Popen(
|
|
277
|
+
[exe, "--folder-uri", uri], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
|
278
|
+
)
|
|
279
|
+
ui.ok(f"opened {exe} on {alias}")
|
|
280
|
+
return True
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def open_shell(alias: str) -> int:
|
|
284
|
+
return subprocess.run(["ssh", alias]).returncode
|
kdev/sshcfg.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Manage the kdev block in ~/.ssh/config.
|
|
2
|
+
|
|
3
|
+
Marker-delimited rewrite rather than parsing Host stanzas: the old
|
|
4
|
+
launch_kaggle.py approach of scanning for the next `Host ` line silently ate
|
|
5
|
+
the final stanza in the file whenever ours was last.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from . import cloudflared
|
|
15
|
+
|
|
16
|
+
BEGIN = "# >>> kdev >>>"
|
|
17
|
+
END = "# <<< kdev <<<"
|
|
18
|
+
BLOCK_RE = re.compile(rf"\n?{re.escape(BEGIN)}.*?{re.escape(END)}\n?", re.DOTALL)
|
|
19
|
+
|
|
20
|
+
SSH_CONFIG = Path.home() / ".ssh" / "config"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def render_block(
|
|
24
|
+
alias: str, hostname: str, user: str = "root", cloudflared_path: Path | None = None
|
|
25
|
+
) -> str:
|
|
26
|
+
proxy = cloudflared.proxy_command(cloudflared_path)
|
|
27
|
+
return (
|
|
28
|
+
f"{BEGIN}\n"
|
|
29
|
+
f"Host {alias}\n"
|
|
30
|
+
f" HostName {hostname}\n"
|
|
31
|
+
f" User {user}\n"
|
|
32
|
+
f" ProxyCommand {proxy}\n"
|
|
33
|
+
# Every session is a new container with a new host key, so a key
|
|
34
|
+
# remembered by one machine is wrong on the next box another machine
|
|
35
|
+
# starts: VS Code and `ssh kaggle` there fail until someone resets it.
|
|
36
|
+
# Only a box holding the workspace's tunnel credentials can answer on
|
|
37
|
+
# this hostname, and whoever holds those can already edit the
|
|
38
|
+
# notebook that builds the box, so a pinned key protects nothing.
|
|
39
|
+
f" UserKnownHostsFile {os.devnull}\n"
|
|
40
|
+
f" StrictHostKeyChecking no\n"
|
|
41
|
+
f" LogLevel ERROR\n"
|
|
42
|
+
f" ServerAliveInterval 30\n"
|
|
43
|
+
f" ServerAliveCountMax 10\n"
|
|
44
|
+
# For `ssh kaggle htop`. kdev's own background ssh calls pass -T: a tty
|
|
45
|
+
# there puts this terminal in raw mode and smears the live board.
|
|
46
|
+
f" RequestTTY yes\n"
|
|
47
|
+
f"{END}\n"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def write(
|
|
52
|
+
alias: str, hostname: str, user: str = "root", cloudflared_path: Path | None = None
|
|
53
|
+
) -> Path:
|
|
54
|
+
SSH_CONFIG.parent.mkdir(mode=0o700, exist_ok=True)
|
|
55
|
+
existing = SSH_CONFIG.read_text() if SSH_CONFIG.exists() else ""
|
|
56
|
+
stripped = BLOCK_RE.sub("\n", existing).strip()
|
|
57
|
+
body = (stripped + "\n\n" if stripped else "") + render_block(
|
|
58
|
+
alias, hostname, user, cloudflared_path
|
|
59
|
+
)
|
|
60
|
+
SSH_CONFIG.write_text(body)
|
|
61
|
+
SSH_CONFIG.chmod(0o600)
|
|
62
|
+
return SSH_CONFIG
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def has_block() -> bool:
|
|
66
|
+
"""Whether ~/.ssh/config currently points the alias at a session."""
|
|
67
|
+
return SSH_CONFIG.exists() and BEGIN in SSH_CONFIG.read_text()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def clear() -> None:
|
|
71
|
+
if SSH_CONFIG.exists():
|
|
72
|
+
SSH_CONFIG.write_text(BLOCK_RE.sub("\n", SSH_CONFIG.read_text()).strip() + "\n")
|
kdev/tunnel.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Create and route a Cloudflare named tunnel entirely from the CLI.
|
|
2
|
+
|
|
3
|
+
Cloudflare has two tunnel modes and they are not interchangeable:
|
|
4
|
+
|
|
5
|
+
* remotely-managed (`run --token`): ingress rules live in the Zero Trust
|
|
6
|
+
dashboard, and a local config file is ignored.
|
|
7
|
+
* locally-managed (`run --credentials-file` + `--config`): ingress comes from
|
|
8
|
+
a config file next to the connector.
|
|
9
|
+
|
|
10
|
+
kdev uses the locally-managed mode. The remote side is a Kaggle container we
|
|
11
|
+
generate from scratch every session, so shipping it a six-line config is easier
|
|
12
|
+
than making the user provision ingress by hand -- and it means the whole setup
|
|
13
|
+
is scriptable, with no Zero Trust onboarding at all.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import base64
|
|
19
|
+
import json
|
|
20
|
+
import re
|
|
21
|
+
import subprocess
|
|
22
|
+
import tempfile
|
|
23
|
+
import urllib.error
|
|
24
|
+
import urllib.parse
|
|
25
|
+
import urllib.request
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from . import cloudflared
|
|
29
|
+
|
|
30
|
+
CERT = Path.home() / ".cloudflared" / "cert.pem"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class TunnelError(RuntimeError):
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _run(args: list[str], **kw) -> subprocess.CompletedProcess:
|
|
38
|
+
cf = cloudflared.ensure(auto=False)
|
|
39
|
+
return subprocess.run([str(cf), *args], text=True, capture_output=True, **kw)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def logged_in() -> bool:
|
|
43
|
+
return CERT.exists()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def login() -> None:
|
|
47
|
+
"""Opens a browser so the user can pick the zone. Writes ~/.cloudflared/cert.pem."""
|
|
48
|
+
cf = cloudflared.ensure(auto=False)
|
|
49
|
+
# Inherit stdio: this prints a URL and waits for the browser round-trip.
|
|
50
|
+
proc = subprocess.run([str(cf), "tunnel", "login"])
|
|
51
|
+
if proc.returncode != 0 or not CERT.exists():
|
|
52
|
+
raise TunnelError("cloudflared login did not complete.")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def list_tunnels() -> list[dict]:
|
|
56
|
+
out = _run(["tunnel", "list", "--output", "json"])
|
|
57
|
+
if out.returncode != 0:
|
|
58
|
+
raise TunnelError(out.stderr.strip() or "could not list tunnels")
|
|
59
|
+
try:
|
|
60
|
+
return json.loads(out.stdout or "[]") or []
|
|
61
|
+
except json.JSONDecodeError:
|
|
62
|
+
return []
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def find_tunnel(name: str) -> dict | None:
|
|
66
|
+
return next((t for t in list_tunnels() if t.get("name") == name), None)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def create(name: str) -> str:
|
|
70
|
+
"""Create the tunnel, or reuse it if the name is already taken. Returns its UUID."""
|
|
71
|
+
existing = find_tunnel(name)
|
|
72
|
+
if existing:
|
|
73
|
+
return existing["id"]
|
|
74
|
+
out = _run(["tunnel", "create", name])
|
|
75
|
+
if out.returncode != 0:
|
|
76
|
+
raise TunnelError(out.stderr.strip() or f"could not create tunnel {name!r}")
|
|
77
|
+
found = find_tunnel(name)
|
|
78
|
+
if not found:
|
|
79
|
+
raise TunnelError(f"created {name!r} but it does not appear in the tunnel list")
|
|
80
|
+
return found["id"]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _api(path: str) -> dict:
|
|
84
|
+
"""Read the Cloudflare API with the token cert.pem already carries."""
|
|
85
|
+
tok = _cert_token()
|
|
86
|
+
if not tok.get("apiToken") or not tok.get("zoneID"):
|
|
87
|
+
return {}
|
|
88
|
+
req = urllib.request.Request(
|
|
89
|
+
f"https://api.cloudflare.com/client/v4/zones/{tok['zoneID']}{path}",
|
|
90
|
+
headers={"Authorization": f"Bearer {tok['apiToken']}"},
|
|
91
|
+
)
|
|
92
|
+
try:
|
|
93
|
+
with urllib.request.urlopen(req, timeout=20) as r:
|
|
94
|
+
return json.load(r)
|
|
95
|
+
except (urllib.error.URLError, ValueError, TimeoutError):
|
|
96
|
+
return {}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def dns_record(hostname: str) -> dict | None:
|
|
100
|
+
"""The existing DNS record for a hostname, if there is one."""
|
|
101
|
+
body = _api(f"/dns_records?name={urllib.parse.quote(hostname)}")
|
|
102
|
+
if not body.get("success"):
|
|
103
|
+
return None
|
|
104
|
+
return next(iter(body.get("result") or []), None)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def tunnel_target(tunnel_id: str) -> str:
|
|
108
|
+
return f"{tunnel_id}.cfargotunnel.com"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def route_dns(name: str, hostname: str, tunnel_id: str = "", on_conflict=None) -> str:
|
|
112
|
+
"""Point hostname at the tunnel, but only if it is not already pointed there.
|
|
113
|
+
|
|
114
|
+
Returns "reused", "created" or "repointed". The previous version passed
|
|
115
|
+
--overwrite-dns unconditionally, which rewrote a correct record on every
|
|
116
|
+
run and would have silently clobbered an unrelated one.
|
|
117
|
+
"""
|
|
118
|
+
target = tunnel_target(tunnel_id) if tunnel_id else ""
|
|
119
|
+
existing = dns_record(hostname) if target else None
|
|
120
|
+
|
|
121
|
+
if existing and existing.get("content") == target:
|
|
122
|
+
return "reused"
|
|
123
|
+
|
|
124
|
+
if existing and on_conflict and not on_conflict(existing):
|
|
125
|
+
raise TunnelError(f"{hostname} already points at {existing.get('content')}")
|
|
126
|
+
|
|
127
|
+
args = ["tunnel", "route", "dns"]
|
|
128
|
+
if existing:
|
|
129
|
+
args.append("--overwrite-dns")
|
|
130
|
+
out = _run([*args, name, hostname])
|
|
131
|
+
if out.returncode != 0:
|
|
132
|
+
msg = (out.stderr or out.stdout).strip()
|
|
133
|
+
if "already exists" not in msg.lower():
|
|
134
|
+
raise TunnelError(msg or f"could not route {hostname}")
|
|
135
|
+
return "repointed" if existing else "created"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def credentials(name: str) -> str:
|
|
139
|
+
"""Fetch the tunnel's credentials JSON, which the Kaggle side runs with."""
|
|
140
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
141
|
+
path = Path(tmp) / "cred.json"
|
|
142
|
+
out = _run(["tunnel", "token", "--cred-file", str(path), name])
|
|
143
|
+
if out.returncode != 0:
|
|
144
|
+
raise TunnelError(out.stderr.strip() or "could not fetch tunnel credentials")
|
|
145
|
+
if not path.exists():
|
|
146
|
+
raise TunnelError("cloudflared reported success but wrote no credentials file")
|
|
147
|
+
blob = path.read_text().strip()
|
|
148
|
+
parsed = json.loads(blob)
|
|
149
|
+
for key in ("AccountTag", "TunnelSecret", "TunnelID"):
|
|
150
|
+
if key not in parsed:
|
|
151
|
+
raise TunnelError(f"credentials JSON is missing {key}")
|
|
152
|
+
return blob
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _cert_token() -> dict:
|
|
156
|
+
"""cert.pem is a PEM-wrapped base64 JSON blob: {accountID, apiToken, zoneID}."""
|
|
157
|
+
if not CERT.exists():
|
|
158
|
+
return {}
|
|
159
|
+
b64 = "".join(ln for ln in CERT.read_text().splitlines() if not ln.startswith("-----"))
|
|
160
|
+
try:
|
|
161
|
+
return json.loads(base64.b64decode(b64))
|
|
162
|
+
except (ValueError, json.JSONDecodeError):
|
|
163
|
+
return {}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def zone_name() -> str:
|
|
167
|
+
"""Resolve the authorised zone's name.
|
|
168
|
+
|
|
169
|
+
cert.pem carries a zone *ID*, not a name, so the name has to be looked up.
|
|
170
|
+
The api token in the same file is scoped for exactly this.
|
|
171
|
+
"""
|
|
172
|
+
tok = _cert_token()
|
|
173
|
+
if not tok.get("zoneID") or not tok.get("apiToken"):
|
|
174
|
+
return ""
|
|
175
|
+
req = urllib.request.Request(
|
|
176
|
+
f"https://api.cloudflare.com/client/v4/zones/{tok['zoneID']}",
|
|
177
|
+
headers={"Authorization": f"Bearer {tok['apiToken']}"},
|
|
178
|
+
)
|
|
179
|
+
try:
|
|
180
|
+
with urllib.request.urlopen(req, timeout=20) as r:
|
|
181
|
+
body = json.load(r)
|
|
182
|
+
return body["result"]["name"] if body.get("success") else ""
|
|
183
|
+
except (urllib.error.URLError, KeyError, ValueError, TimeoutError):
|
|
184
|
+
return ""
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def validate_label(label: str) -> str:
|
|
188
|
+
"""A single DNS label: what the box is called, and its subdomain."""
|
|
189
|
+
label = label.strip().lower()
|
|
190
|
+
if not re.fullmatch(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?", label or ""):
|
|
191
|
+
raise TunnelError(f"{label!r} is not a valid name. Use letters, digits and hyphens.")
|
|
192
|
+
return label
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def validate_hostname(hostname: str, zone: str = "") -> str:
|
|
196
|
+
"""Reject anything that is not a real subdomain of the authorised zone.
|
|
197
|
+
|
|
198
|
+
A bare label like `kdev` looks plausible next to the tunnel-name prompt but
|
|
199
|
+
is not routable: `tunnel route dns` needs an FQDN inside a zone you own.
|
|
200
|
+
"""
|
|
201
|
+
hostname = hostname.strip().lower().rstrip(".")
|
|
202
|
+
if not hostname:
|
|
203
|
+
raise TunnelError("Hostname cannot be empty.")
|
|
204
|
+
if "." not in hostname:
|
|
205
|
+
raise TunnelError(
|
|
206
|
+
f"{hostname!r} is a bare label, not a hostname. "
|
|
207
|
+
f"Use a subdomain such as kaggle.{zone or 'yourdomain.com'}"
|
|
208
|
+
)
|
|
209
|
+
if not re.fullmatch(
|
|
210
|
+
r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+", hostname
|
|
211
|
+
):
|
|
212
|
+
raise TunnelError(f"{hostname!r} is not a valid DNS name.")
|
|
213
|
+
if zone and not (hostname == zone or hostname.endswith("." + zone)):
|
|
214
|
+
raise TunnelError(
|
|
215
|
+
f"{hostname!r} is not inside your Cloudflare zone {zone!r}. Try kaggle.{zone}"
|
|
216
|
+
)
|
|
217
|
+
return hostname
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def provision(name: str, hostname: str, on_conflict=None) -> tuple[str, str, str]:
|
|
221
|
+
"""Create, route and fetch credentials. Returns (id, creds_json, dns_status)."""
|
|
222
|
+
tunnel_id = create(name)
|
|
223
|
+
status = route_dns(name, hostname, tunnel_id, on_conflict)
|
|
224
|
+
return tunnel_id, credentials(name), status
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def existing_for_zone(zone: str) -> list[dict]:
|
|
228
|
+
"""Tunnels on this account that already have a DNS record in this zone.
|
|
229
|
+
|
|
230
|
+
Used to offer a reuse instead of asking for a name and creating a duplicate.
|
|
231
|
+
"""
|
|
232
|
+
if not zone:
|
|
233
|
+
return []
|
|
234
|
+
try:
|
|
235
|
+
candidates = list_tunnels()
|
|
236
|
+
except TunnelError:
|
|
237
|
+
return []
|
|
238
|
+
out = []
|
|
239
|
+
for t in candidates:
|
|
240
|
+
host = f"{t.get('name')}.{zone}"
|
|
241
|
+
rec = dns_record(host)
|
|
242
|
+
if rec and rec.get("content") == tunnel_target(t.get("id", "")):
|
|
243
|
+
out.append({"name": t.get("name"), "id": t.get("id"), "hostname": host})
|
|
244
|
+
return out
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def is_configured(cfg) -> bool:
|
|
248
|
+
"""True when the saved tunnel still exists on the account and DNS matches."""
|
|
249
|
+
if not (cfg.tunnel_credentials and cfg.tunnel_id and cfg.tunnel_hostname):
|
|
250
|
+
return False
|
|
251
|
+
if not logged_in():
|
|
252
|
+
return False
|
|
253
|
+
try:
|
|
254
|
+
if not any(t.get("id") == cfg.tunnel_id for t in list_tunnels()):
|
|
255
|
+
return False
|
|
256
|
+
except TunnelError:
|
|
257
|
+
return False
|
|
258
|
+
rec = dns_record(cfg.tunnel_hostname)
|
|
259
|
+
return bool(rec and rec.get("content") == tunnel_target(cfg.tunnel_id))
|