kdev-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kdev/__init__.py +7 -0
- kdev/__main__.py +3 -0
- kdev/api.py +408 -0
- kdev/auth.py +115 -0
- kdev/bootstrap.py +510 -0
- kdev/cli.py +162 -0
- kdev/cloudflared.py +179 -0
- kdev/commands/__init__.py +5 -0
- kdev/commands/_common.py +51 -0
- kdev/commands/account.py +148 -0
- kdev/commands/box.py +694 -0
- kdev/commands/doctor.py +127 -0
- kdev/commands/home.py +92 -0
- kdev/commands/settings.py +164 -0
- kdev/commands/tunnel.py +78 -0
- kdev/commands/workspace.py +210 -0
- kdev/config.py +181 -0
- kdev/errors.py +25 -0
- kdev/kdev_box.py +538 -0
- kdev/notebook.py +261 -0
- kdev/persistence.py +314 -0
- kdev/quota.py +102 -0
- kdev/safety.py +72 -0
- kdev/session.py +284 -0
- kdev/sshcfg.py +72 -0
- kdev/tunnel.py +259 -0
- kdev/ui.py +666 -0
- kdev/wizard.py +251 -0
- kdev_cli-0.1.3.dist-info/METADATA +598 -0
- kdev_cli-0.1.3.dist-info/RECORD +33 -0
- kdev_cli-0.1.3.dist-info/WHEEL +4 -0
- kdev_cli-0.1.3.dist-info/entry_points.txt +2 -0
- kdev_cli-0.1.3.dist-info/licenses/LICENSE +21 -0
kdev/notebook.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""The shared notebook: the one box every account in the group runs.
|
|
2
|
+
|
|
3
|
+
A Kaggle group cannot own anything -- "groups can never own resources", in
|
|
4
|
+
Kaggle's own words. So one account owns the notebook and shares it with the
|
|
5
|
+
group as Editor (Kaggle's "Can Edit"). Every member can then run it on their
|
|
6
|
+
own quota (measured: a session is charged to the account that starts it) and
|
|
7
|
+
read every version it has saved, which is what carries your files between
|
|
8
|
+
accounts.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
|
|
15
|
+
from . import api, bootstrap, config, persistence, ui
|
|
16
|
+
from .errors import KdevError
|
|
17
|
+
|
|
18
|
+
PLACEHOLDER = "# kdev workspace notebook -- source is generated per session.\n"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def notebook_url(slug: str) -> str:
|
|
22
|
+
return f"https://www.kaggle.com/code/{slug}/edit"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def slugify(title: str) -> str:
|
|
26
|
+
"""Kaggle's rule: the slug is the title, lowercased, dashes for the rest."""
|
|
27
|
+
return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def valid_ref(ref: str) -> str:
|
|
31
|
+
ref = (ref or "").strip().strip("/")
|
|
32
|
+
if not re.fullmatch(r"[A-Za-z0-9_.-]+/[a-z0-9][a-z0-9-]*", ref):
|
|
33
|
+
raise KdevError(
|
|
34
|
+
f"{ref!r} is not a notebook reference.",
|
|
35
|
+
"Use owner/notebook-slug, as in kaggle.com/code/owner/notebook-slug",
|
|
36
|
+
)
|
|
37
|
+
return ref
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# --- reading ------------------------------------------------------------------
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def discover(creds: api.Creds) -> list[tuple[str, str]]:
|
|
44
|
+
"""Notebooks shared with this account."""
|
|
45
|
+
return [
|
|
46
|
+
(k["ref"], k.get("title") or k["ref"])
|
|
47
|
+
for k in api.list_shared_kernels(creds)
|
|
48
|
+
if k.get("ref")
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def scan(creds: api.Creds) -> tuple[list[dict], list[dict]]:
|
|
53
|
+
"""Which groups this account is in, and which notebooks each shares.
|
|
54
|
+
|
|
55
|
+
Kaggle has no API that lists your groups (they are private and
|
|
56
|
+
invitation-only), so they are derived from the notebooks shared with you
|
|
57
|
+
and the sharing policy on each. A group with no shared notebook is
|
|
58
|
+
therefore invisible, which is why creating the first box asks for a slug.
|
|
59
|
+
"""
|
|
60
|
+
notebooks = []
|
|
61
|
+
for ref, title in discover(creds):
|
|
62
|
+
kid = api.kernel_id(creds, ref)
|
|
63
|
+
groups = api.policy_groups(api.get_policy(creds, kid)) if kid else []
|
|
64
|
+
notebooks.append({"ref": ref, "title": title, "id": kid, "groups": groups})
|
|
65
|
+
by_slug: dict[str, dict] = {}
|
|
66
|
+
for nb in notebooks:
|
|
67
|
+
for g in nb["groups"]:
|
|
68
|
+
by_slug.setdefault(g["slug"], {**g, "notebooks": []})["notebooks"].append(nb)
|
|
69
|
+
return list(by_slug.values()), notebooks
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def describe(cfg: config.Config, creds: api.Creds) -> dict:
|
|
73
|
+
"""Everything `kdev workspace` shows, as data."""
|
|
74
|
+
out: dict = {
|
|
75
|
+
"notebook": cfg.notebook,
|
|
76
|
+
"group": cfg.group_slug,
|
|
77
|
+
"role": "",
|
|
78
|
+
"members": [],
|
|
79
|
+
"version": 0,
|
|
80
|
+
"files": None,
|
|
81
|
+
"errors": [],
|
|
82
|
+
}
|
|
83
|
+
if not cfg.notebook:
|
|
84
|
+
return out
|
|
85
|
+
try:
|
|
86
|
+
meta = api.get_kernel(creds, cfg.notebook)
|
|
87
|
+
out["version"] = int(meta.get("currentVersionNumber") or 0)
|
|
88
|
+
out["kernel_type"] = meta.get("kernelType", "")
|
|
89
|
+
cfg.notebook_id = cfg.notebook_id or int(meta.get("id") or 0)
|
|
90
|
+
except (api.KaggleError, TypeError, ValueError) as e:
|
|
91
|
+
out["errors"].append(f"notebook: {e}")
|
|
92
|
+
if cfg.group_slug and cfg.notebook_id:
|
|
93
|
+
try:
|
|
94
|
+
out["role"] = api.group_role(api.get_policy(creds, cfg.notebook_id), cfg.group_slug)
|
|
95
|
+
except api.KaggleError as e:
|
|
96
|
+
out["errors"].append(f"sharing: {e}")
|
|
97
|
+
try:
|
|
98
|
+
out["members"] = [
|
|
99
|
+
m.get("username", "?") for m in api.group_members(creds, cfg.group_slug)
|
|
100
|
+
]
|
|
101
|
+
except api.KaggleError as e:
|
|
102
|
+
out["errors"].append(f"members: {e}")
|
|
103
|
+
# The newest version with files, not the newest version: while a box runs
|
|
104
|
+
# its own has none yet, and "0 files" reads as everything being lost.
|
|
105
|
+
out["saved"], files = persistence.newest_saved(creds, cfg.notebook, out["version"])
|
|
106
|
+
out["files"] = len(files) if out["saved"] else None
|
|
107
|
+
return out
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def adopt_config(cfg: config.Config, creds: api.Creds) -> list[str]:
|
|
111
|
+
"""Take the tunnel and git settings the notebook already runs with.
|
|
112
|
+
|
|
113
|
+
A second laptop joining the workspace would otherwise redo the whole
|
|
114
|
+
Cloudflare setup to reach a box the first laptop already knows how to
|
|
115
|
+
reach. Only fills what is empty here, so a machine's own explicit settings
|
|
116
|
+
always win. Returns the names of the settings it took.
|
|
117
|
+
"""
|
|
118
|
+
if not cfg.notebook:
|
|
119
|
+
return []
|
|
120
|
+
try:
|
|
121
|
+
found = bootstrap.extract_config(api.kernel_source(creds, cfg.notebook))
|
|
122
|
+
except api.KaggleError:
|
|
123
|
+
return []
|
|
124
|
+
has_tunnel = bool(cfg.tunnel_credentials or cfg.tunnel_token)
|
|
125
|
+
taken = []
|
|
126
|
+
for key, value in found.items():
|
|
127
|
+
if key.startswith("tunnel_") and has_tunnel:
|
|
128
|
+
continue
|
|
129
|
+
if not getattr(cfg, key):
|
|
130
|
+
setattr(cfg, key, value)
|
|
131
|
+
taken.append(key)
|
|
132
|
+
return taken
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# Kept under its earlier name for callers written against it.
|
|
136
|
+
adopt_notebook_config = adopt_config
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# --- changing -----------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def create(creds: api.Creds, owner: str, title: str) -> tuple[str, int]:
|
|
143
|
+
"""Create the workspace notebook. Never over an existing one.
|
|
144
|
+
|
|
145
|
+
SaveKernel on a name that already exists replaces its source -- so a typo
|
|
146
|
+
that matched a real notebook would wipe it. Check first.
|
|
147
|
+
"""
|
|
148
|
+
slug = f"{owner}/{slugify(title)}"
|
|
149
|
+
if not slugify(title):
|
|
150
|
+
raise KdevError(f"{title!r} makes an empty notebook name.", "Use letters or digits.")
|
|
151
|
+
try:
|
|
152
|
+
existing = api.get_kernel(creds, slug)
|
|
153
|
+
except api.KaggleError:
|
|
154
|
+
existing = {}
|
|
155
|
+
if existing.get("id"):
|
|
156
|
+
raise KdevError(
|
|
157
|
+
f"{slug} already exists.",
|
|
158
|
+
f"Use it with: kdev workspace use {slug} or pick another --name",
|
|
159
|
+
)
|
|
160
|
+
resp = api.save_kernel(
|
|
161
|
+
creds,
|
|
162
|
+
slug=slug,
|
|
163
|
+
title=title,
|
|
164
|
+
source=PLACEHOLDER,
|
|
165
|
+
machine_shape=None,
|
|
166
|
+
timeout_seconds=600,
|
|
167
|
+
run=False,
|
|
168
|
+
)
|
|
169
|
+
kernel_id = int(resp.get("kernelId") or api.get_kernel(creds, slug).get("id") or 0)
|
|
170
|
+
if not kernel_id:
|
|
171
|
+
raise KdevError(
|
|
172
|
+
f"Created {slug} but could not read its id back.",
|
|
173
|
+
"Retry sharing with: kdev workspace share",
|
|
174
|
+
)
|
|
175
|
+
return slug, kernel_id
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def share(creds: api.Creds, kernel_id: int, group_slug: str) -> None:
|
|
179
|
+
try:
|
|
180
|
+
api.share_with_group(creds, kernel_id, group_slug)
|
|
181
|
+
except api.KaggleError as e:
|
|
182
|
+
raise KdevError(
|
|
183
|
+
f"Could not share with the group {group_slug!r}: {e}",
|
|
184
|
+
"Check the slug (kaggle.com/groups/<slug>) and that the owner is a member.",
|
|
185
|
+
) from e
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def use(cfg: config.Config, creds: api.Creds, ref: str) -> None:
|
|
189
|
+
"""Point this machine at an existing notebook."""
|
|
190
|
+
ref = valid_ref(ref)
|
|
191
|
+
kid = api.kernel_id(creds, ref)
|
|
192
|
+
if not kid:
|
|
193
|
+
raise KdevError(
|
|
194
|
+
f"Could not open {ref}.",
|
|
195
|
+
"Is it shared with this account? kdev workspace join lists what is.",
|
|
196
|
+
)
|
|
197
|
+
groups = api.policy_groups(api.get_policy(creds, kid))
|
|
198
|
+
cfg.notebook, cfg.notebook_id = ref, kid
|
|
199
|
+
cfg.group_slug = groups[0]["slug"] if groups else ""
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def join(cfg: config.Config, creds: api.Creds, owner: str) -> bool:
|
|
203
|
+
"""Interactive: pick the group, then a notebook in it or a new one."""
|
|
204
|
+
from . import safety
|
|
205
|
+
|
|
206
|
+
with ui.spinner("reading your groups and notebooks…"):
|
|
207
|
+
groups, _ = scan(creds)
|
|
208
|
+
|
|
209
|
+
if not groups:
|
|
210
|
+
ui.warn("No group shares a notebook with this account yet.")
|
|
211
|
+
ui.hint("A group's slug is the end of its address: kaggle.com/groups/<slug>")
|
|
212
|
+
slug = ui.ask("Group slug", default=cfg.group_slug)
|
|
213
|
+
if not slug:
|
|
214
|
+
return False
|
|
215
|
+
return _create_and_share(cfg, creds, owner, slug)
|
|
216
|
+
|
|
217
|
+
if len(groups) == 1:
|
|
218
|
+
group = groups[0]
|
|
219
|
+
ui.ok(f"group {group['name']} ({group['members']} members)")
|
|
220
|
+
else:
|
|
221
|
+
pick = ui.choose(
|
|
222
|
+
"Which group?",
|
|
223
|
+
[
|
|
224
|
+
(
|
|
225
|
+
g["slug"],
|
|
226
|
+
f"{g['name']:<24} {g['members']} members · {len(g['notebooks'])} notebook(s)",
|
|
227
|
+
)
|
|
228
|
+
for g in groups
|
|
229
|
+
],
|
|
230
|
+
)
|
|
231
|
+
group = next(g for g in groups if g["slug"] == pick)
|
|
232
|
+
|
|
233
|
+
options = []
|
|
234
|
+
for nb in group["notebooks"]:
|
|
235
|
+
try:
|
|
236
|
+
verdict, _, _ = safety.check(creds, nb["ref"])
|
|
237
|
+
except api.KaggleError:
|
|
238
|
+
verdict = "occupied"
|
|
239
|
+
tag = "kdev box" if verdict == "safe" else "has its own code (kept below kdev's cell)"
|
|
240
|
+
options.append((nb["ref"], f"{nb['title']:<28} {tag}"))
|
|
241
|
+
options.append(("", "+ New notebook for this group"))
|
|
242
|
+
pick = ui.choose(f"Which notebook in {group['name']}?", options) if len(options) > 1 else ""
|
|
243
|
+
if pick:
|
|
244
|
+
cfg.notebook = pick
|
|
245
|
+
cfg.notebook_id = next(nb["id"] for nb in group["notebooks"] if nb["ref"] == pick)
|
|
246
|
+
cfg.group_slug = group["slug"]
|
|
247
|
+
ui.ok(f"workspace {cfg.notebook}")
|
|
248
|
+
return True
|
|
249
|
+
return _create_and_share(cfg, creds, owner, group["slug"])
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _create_and_share(cfg: config.Config, creds: api.Creds, owner: str, group_slug: str) -> bool:
|
|
253
|
+
title = ui.ask("Name for the new notebook", default="kdev-box")
|
|
254
|
+
with ui.spinner("creating the notebook…"):
|
|
255
|
+
slug, kid = create(creds, owner, title)
|
|
256
|
+
ui.ok(f"created {slug}")
|
|
257
|
+
with ui.spinner(f"sharing with {group_slug}…"):
|
|
258
|
+
share(creds, kid, group_slug)
|
|
259
|
+
ui.ok(f"shared with {group_slug} (Can Edit)")
|
|
260
|
+
cfg.notebook, cfg.notebook_id, cfg.group_slug = slug, kid, group_slug
|
|
261
|
+
return True
|
kdev/persistence.py
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Where your workspace lives between sessions, and how it gets back.
|
|
2
|
+
|
|
3
|
+
The notebook's saved versions are the source of truth. Kaggle commits
|
|
4
|
+
/kaggle/working as a version whenever a batch run ends -- a clean exit, a crash
|
|
5
|
+
and a cancel were all measured to commit, up to the moment the container
|
|
6
|
+
stopped -- and every version stays readable afterwards. That store belongs to
|
|
7
|
+
the *notebook*, not to an account or a machine, so any account in the group,
|
|
8
|
+
on any laptop, sees the same thing.
|
|
9
|
+
|
|
10
|
+
kdev's job is to put it back. It finds the right saved version here, where the
|
|
11
|
+
Kaggle login lives, and hands the box a plan; the box downloads straight from
|
|
12
|
+
Kaggle (see kdev_box.py). Nothing routes through your laptop, and nothing on
|
|
13
|
+
your laptop can override it.
|
|
14
|
+
|
|
15
|
+
The local mirror is a backup and nothing more: `kdev backup` copies a live box
|
|
16
|
+
down, `kdev restore --from-backup` pushes it back up on request.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import subprocess
|
|
23
|
+
import time
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
import httpx
|
|
27
|
+
|
|
28
|
+
from . import api
|
|
29
|
+
from . import config as _config
|
|
30
|
+
|
|
31
|
+
REMOTE = "/kaggle/working"
|
|
32
|
+
|
|
33
|
+
#: Kept out of the local backup. `.kdev` is the live session's own state:
|
|
34
|
+
#: pushing an old copy back up would tell the box it is restored when it is not.
|
|
35
|
+
#: A leading / anchors a name to the top, where the box writes it; a project's
|
|
36
|
+
#: own web/static/custom.css is still backed up.
|
|
37
|
+
EXCLUDES = [
|
|
38
|
+
"/cloudflared.log",
|
|
39
|
+
"/.kdev-session",
|
|
40
|
+
"/.kdev-stop",
|
|
41
|
+
"/.kdev",
|
|
42
|
+
"/__notebook__.ipynb",
|
|
43
|
+
"/__output__.json",
|
|
44
|
+
"/custom.css",
|
|
45
|
+
".virtual_documents",
|
|
46
|
+
".ipynb_checkpoints",
|
|
47
|
+
"__pycache__",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def mirror_dir() -> Path:
|
|
52
|
+
"""Where the workspace lives on this machine.
|
|
53
|
+
|
|
54
|
+
One directory for everything, deliberately NOT keyed to the notebook:
|
|
55
|
+
notebooks get recreated and switched constantly, and keying the mirror to
|
|
56
|
+
them meant every new notebook handed you an empty workspace -- which looked
|
|
57
|
+
exactly like the sync being broken.
|
|
58
|
+
"""
|
|
59
|
+
return _config.CONFIG_DIR / "workspace"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
STATE = ".kdev/state.json"
|
|
63
|
+
FETCHED = ".kdev/fetched"
|
|
64
|
+
|
|
65
|
+
#: How far back to look for a version to build on. A handful of cut-short
|
|
66
|
+
#: sessions in a row is already unusual; ten means something else is wrong.
|
|
67
|
+
WALK_BACK = 10
|
|
68
|
+
#: Layers stacked when the newest version was never fully restored. Each one
|
|
69
|
+
#: is a complete listing, so this bounds the plan's size.
|
|
70
|
+
MAX_LAYERS = 4
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def version_files(creds: api.Creds, notebook: str, label: str) -> dict[str, str] | None:
|
|
74
|
+
"""name -> signed URL for one saved version, or None if there is none."""
|
|
75
|
+
try:
|
|
76
|
+
files = api.session_output(creds, notebook, label)
|
|
77
|
+
except api.KaggleError:
|
|
78
|
+
return None
|
|
79
|
+
return {f["name"]: f["url"] for f in files}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _state(files: dict[str, str]) -> dict | None:
|
|
83
|
+
"""The .kdev/state.json a version saved, fetched through its own URL."""
|
|
84
|
+
url = files.get(STATE)
|
|
85
|
+
if not url:
|
|
86
|
+
return None
|
|
87
|
+
try:
|
|
88
|
+
r = httpx.get(url, follow_redirects=True, timeout=30)
|
|
89
|
+
data = r.json() if r.status_code == 200 else None
|
|
90
|
+
except (httpx.HTTPError, ValueError):
|
|
91
|
+
return None
|
|
92
|
+
return data if isinstance(data, dict) else None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def plan_layers(creds: api.Creds, notebook: str, latest: int) -> list[str]:
|
|
96
|
+
"""Which saved versions, oldest first, rebuild the workspace as you left it.
|
|
97
|
+
|
|
98
|
+
Walks back from `latest`:
|
|
99
|
+
* a version with no files was not a working session (a Quick Save of the
|
|
100
|
+
source, the placeholder `workspace create` made) -- skip it;
|
|
101
|
+
* a version whose state says it was fully restored, or that kdev never
|
|
102
|
+
touched, is complete on its own -- use just that;
|
|
103
|
+
* a version that was never fully restored (killed mid-restore, or nobody
|
|
104
|
+
ran the restore before working in it) still holds whatever was done in
|
|
105
|
+
it -- stack it on the layers it was meant to be built on.
|
|
106
|
+
"""
|
|
107
|
+
label, files = newest_saved(creds, notebook, latest)
|
|
108
|
+
if not label:
|
|
109
|
+
return []
|
|
110
|
+
state = _state(files)
|
|
111
|
+
if state and (state.get("restored") is False or state.get("missing")):
|
|
112
|
+
base = [str(x) for x in state.get("layers") or []]
|
|
113
|
+
return [*base, label][-MAX_LAYERS:]
|
|
114
|
+
return [label]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def newest_saved(creds: api.Creds, notebook: str, latest: int) -> tuple[str, dict[str, str]]:
|
|
118
|
+
"""The newest version, from `latest` back, that holds any files.
|
|
119
|
+
|
|
120
|
+
A running session's own version has none until it ends, and neither has a
|
|
121
|
+
Quick Save of the source: neither is where your files are.
|
|
122
|
+
"""
|
|
123
|
+
for number in range(latest, max(0, latest - WALK_BACK), -1):
|
|
124
|
+
files = version_files(creds, notebook, f"v{number}")
|
|
125
|
+
if files:
|
|
126
|
+
return f"v{number}", files
|
|
127
|
+
return "", {}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _fetched(files: dict[str, str]) -> set[str]:
|
|
131
|
+
"""The names a version's restore delivered (its .kdev/fetched)."""
|
|
132
|
+
url = files.get(FETCHED)
|
|
133
|
+
if not url:
|
|
134
|
+
return set()
|
|
135
|
+
try:
|
|
136
|
+
r = httpx.get(url, follow_redirects=True, timeout=60)
|
|
137
|
+
lines = r.text.splitlines() if r.status_code == 200 else []
|
|
138
|
+
return {json.loads(line) for line in lines if line.strip()}
|
|
139
|
+
except (httpx.HTTPError, ValueError):
|
|
140
|
+
return set()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _still_needed(top: dict[str, str]):
|
|
144
|
+
"""Which files of the layers under `top` it still needs, or None for all.
|
|
145
|
+
|
|
146
|
+
A layer is stacked under the newest only because the newest never got all
|
|
147
|
+
of it. Taking the rest too would bring back whatever was deleted since.
|
|
148
|
+
"""
|
|
149
|
+
state = _state(top) or {}
|
|
150
|
+
if state.get("restored") and state.get("missing"):
|
|
151
|
+
return set(state["missing"]).__contains__
|
|
152
|
+
if state.get("restored") is False:
|
|
153
|
+
got = _fetched(top)
|
|
154
|
+
if got:
|
|
155
|
+
return lambda name: name not in got
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def build_plan(creds: api.Creds, notebook: str, labels: list[str]) -> dict:
|
|
160
|
+
"""Fresh signed URLs for each layer, resolved just before they are used."""
|
|
161
|
+
listed = [(label, version_files(creds, notebook, label)) for label in labels]
|
|
162
|
+
listed = [(label, files) for label, files in listed if files]
|
|
163
|
+
needed = _still_needed(listed[-1][1]) if len(listed) > 1 else None
|
|
164
|
+
layers = []
|
|
165
|
+
for i, (label, files) in enumerate(listed):
|
|
166
|
+
if needed and i < len(listed) - 1:
|
|
167
|
+
# .kdev/ carries the metadata and keys every layer contributes.
|
|
168
|
+
files = {n: u for n, u in files.items() if n.startswith(".kdev/") or needed(n)}
|
|
169
|
+
layers.append({"label": label, "files": files})
|
|
170
|
+
return {"layers": layers}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _ssh(
|
|
174
|
+
alias: str, command: str, stdin: str | None = None, timeout: int = 120
|
|
175
|
+
) -> subprocess.CompletedProcess:
|
|
176
|
+
return subprocess.run(
|
|
177
|
+
["ssh", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=20", alias, command],
|
|
178
|
+
input=stdin,
|
|
179
|
+
capture_output=True,
|
|
180
|
+
text=True,
|
|
181
|
+
timeout=timeout,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def box_state(alias: str) -> dict:
|
|
186
|
+
"""The live session's .kdev/state.json, read over ssh."""
|
|
187
|
+
try:
|
|
188
|
+
r = _ssh(alias, f"cat {REMOTE}/{STATE}", timeout=40)
|
|
189
|
+
except (subprocess.SubprocessError, OSError):
|
|
190
|
+
return {}
|
|
191
|
+
try:
|
|
192
|
+
data = json.loads(r.stdout) if r.returncode == 0 else {}
|
|
193
|
+
except ValueError:
|
|
194
|
+
return {}
|
|
195
|
+
return data if isinstance(data, dict) else {}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def restore_on_box(alias: str, plan: dict, progress=None) -> tuple[bool, dict]:
|
|
199
|
+
"""Start the restore on the box, then follow it until it finishes.
|
|
200
|
+
|
|
201
|
+
The box runs it detached, so if this process dies -- laptop asleep, wifi
|
|
202
|
+
gone -- the restore carries on, and `kdev restore` picks the progress up
|
|
203
|
+
again from any machine.
|
|
204
|
+
"""
|
|
205
|
+
from . import bootstrap
|
|
206
|
+
|
|
207
|
+
if not wait_reachable(alias):
|
|
208
|
+
return False, {"error": "ssh never became reachable"}
|
|
209
|
+
payload = json.dumps({"script": bootstrap.BOX_SCRIPT.read_text(), "plan": plan})
|
|
210
|
+
try:
|
|
211
|
+
r = _ssh(alias, "/root/.kdev/run --receive", stdin=payload, timeout=300)
|
|
212
|
+
except (subprocess.SubprocessError, OSError) as e:
|
|
213
|
+
return False, {"error": str(e)}
|
|
214
|
+
if r.returncode != 0:
|
|
215
|
+
return False, {"error": (r.stderr or r.stdout).strip()[-300:]}
|
|
216
|
+
return follow(alias, progress)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def follow(alias: str, progress=None) -> tuple[bool, dict]:
|
|
220
|
+
"""Stream a running restore's progress. Returns (restored, last state)."""
|
|
221
|
+
last: dict = {}
|
|
222
|
+
try:
|
|
223
|
+
proc = subprocess.Popen(
|
|
224
|
+
[
|
|
225
|
+
"ssh",
|
|
226
|
+
"-T",
|
|
227
|
+
"-o",
|
|
228
|
+
"BatchMode=yes",
|
|
229
|
+
"-o",
|
|
230
|
+
"ConnectTimeout=20",
|
|
231
|
+
"-o",
|
|
232
|
+
"ServerAliveInterval=15",
|
|
233
|
+
alias,
|
|
234
|
+
"/root/.kdev/run --watch",
|
|
235
|
+
],
|
|
236
|
+
stdout=subprocess.PIPE,
|
|
237
|
+
stderr=subprocess.PIPE,
|
|
238
|
+
text=True,
|
|
239
|
+
)
|
|
240
|
+
except OSError as e:
|
|
241
|
+
return False, {"error": str(e)}
|
|
242
|
+
for line in proc.stdout or []:
|
|
243
|
+
try:
|
|
244
|
+
last = json.loads(line)
|
|
245
|
+
except ValueError:
|
|
246
|
+
continue
|
|
247
|
+
if progress:
|
|
248
|
+
progress(last)
|
|
249
|
+
proc.wait()
|
|
250
|
+
return bool(last.get("restored")), last
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _rsync(src: str, dst: str, *extra: str, excludes: bool = True) -> subprocess.CompletedProcess:
|
|
254
|
+
cmd = ["rsync", "-az", "--partial", *extra, "-e", "ssh -o BatchMode=yes -o ConnectTimeout=20"]
|
|
255
|
+
if excludes:
|
|
256
|
+
for pattern in EXCLUDES:
|
|
257
|
+
cmd += ["--exclude", pattern]
|
|
258
|
+
cmd += [src, dst]
|
|
259
|
+
return subprocess.run(cmd, capture_output=True, text=True, timeout=1800)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def wait_reachable(alias: str, timeout: int = 180) -> bool:
|
|
263
|
+
"""Block until ssh can actually open a channel.
|
|
264
|
+
|
|
265
|
+
The tunnel announces itself the moment cloudflared connects, but sshd on
|
|
266
|
+
the far side may not be accepting yet -- pushing straight away failed
|
|
267
|
+
silently and left the box empty.
|
|
268
|
+
"""
|
|
269
|
+
deadline = time.time() + timeout
|
|
270
|
+
while time.time() < deadline:
|
|
271
|
+
r = subprocess.run(
|
|
272
|
+
["ssh", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", alias, "true"],
|
|
273
|
+
capture_output=True,
|
|
274
|
+
)
|
|
275
|
+
if r.returncode == 0:
|
|
276
|
+
return True
|
|
277
|
+
time.sleep(5)
|
|
278
|
+
return False
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def push(alias: str, attempts: int = 3) -> tuple[bool, str]:
|
|
282
|
+
"""Send the local mirror up to the session."""
|
|
283
|
+
local = mirror_dir()
|
|
284
|
+
local.mkdir(parents=True, exist_ok=True)
|
|
285
|
+
if not any(local.iterdir()):
|
|
286
|
+
return True, "nothing saved yet"
|
|
287
|
+
if not wait_reachable(alias):
|
|
288
|
+
return False, "ssh never became reachable"
|
|
289
|
+
err = ""
|
|
290
|
+
for attempt in range(attempts):
|
|
291
|
+
# Pushing a backup fills in; it never replaces what this session wrote.
|
|
292
|
+
r = _rsync(f"{local}/", f"{alias}:{REMOTE}/", "--ignore-existing")
|
|
293
|
+
if r.returncode == 0:
|
|
294
|
+
return True, ""
|
|
295
|
+
err = (r.stderr or "").strip()[-200:]
|
|
296
|
+
time.sleep(3 * (attempt + 1))
|
|
297
|
+
return False, err
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def pull(alias: str) -> tuple[bool, str]:
|
|
301
|
+
"""Bring the session's working directory down into the mirror."""
|
|
302
|
+
local = mirror_dir()
|
|
303
|
+
local.mkdir(parents=True, exist_ok=True)
|
|
304
|
+
# An exact copy of the box: without --delete, files removed on the box --
|
|
305
|
+
# or left from another notebook -- linger here and get pushed back up.
|
|
306
|
+
r = _rsync(f"{alias}:{REMOTE}/", f"{local}/", "--delete")
|
|
307
|
+
return r.returncode == 0, (r.stderr or "").strip()[-200:]
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def contents() -> list[str]:
|
|
311
|
+
local = mirror_dir()
|
|
312
|
+
if not local.is_dir():
|
|
313
|
+
return []
|
|
314
|
+
return sorted(str(p.relative_to(local)) for p in local.rglob("*") if p.is_file())
|
kdev/quota.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Weekly accelerator quota, per account, and choosing who runs a session.
|
|
2
|
+
|
|
3
|
+
Every account in the group carries its own allowance (measured: a session on
|
|
4
|
+
the shared notebook is charged to the account that starts it, not the owner),
|
|
5
|
+
so running short is a switch, not a wall.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
11
|
+
|
|
12
|
+
from . import api, config, ui
|
|
13
|
+
from .errors import KdevError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def secs(value) -> int:
|
|
17
|
+
"""Durations arrive as protobuf Duration strings ('108000s') or numbers."""
|
|
18
|
+
if value is None:
|
|
19
|
+
return 0
|
|
20
|
+
if isinstance(value, (int, float)):
|
|
21
|
+
return int(value)
|
|
22
|
+
try:
|
|
23
|
+
return int(float(str(value).rstrip("s") or 0))
|
|
24
|
+
except ValueError:
|
|
25
|
+
return 0
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def remaining(quota: dict, kind: str = "gpuQuota") -> tuple[int, int]:
|
|
29
|
+
"""(seconds left, seconds allowed) for one quota block."""
|
|
30
|
+
q = quota.get(kind) or {}
|
|
31
|
+
total = secs(q.get("totalTimeAllowed"))
|
|
32
|
+
used = secs(q.get("timeUsed")) + secs(q.get("timeReserved"))
|
|
33
|
+
return max(total - used, 0), total
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _row(cfg: config.Config, name: str) -> ui.ProfileRow:
|
|
37
|
+
prof = cfg.profiles[name]
|
|
38
|
+
row = ui.ProfileRow(
|
|
39
|
+
name=name, username=prof.username, gpu_left=0, gpu_total=0, active=name == cfg.active
|
|
40
|
+
)
|
|
41
|
+
try:
|
|
42
|
+
q = api.quota(prof.creds)
|
|
43
|
+
except api.KaggleError as e:
|
|
44
|
+
row.error = "signed out: kdev account add " + name if e.status == 401 else str(e)[:48]
|
|
45
|
+
return row
|
|
46
|
+
row.gpu_left, row.gpu_total = remaining(q)
|
|
47
|
+
row.tpu_left, row.tpu_total = remaining(q, "tpuQuota")
|
|
48
|
+
return row
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def rows(cfg: config.Config) -> list[ui.ProfileRow]:
|
|
52
|
+
"""Every account's quota, read in parallel -- one slow account must not
|
|
53
|
+
make the whole screen wait N times over."""
|
|
54
|
+
names = list(cfg.profiles)
|
|
55
|
+
if not names:
|
|
56
|
+
return []
|
|
57
|
+
with ThreadPoolExecutor(max_workers=min(8, len(names))) as pool:
|
|
58
|
+
return list(pool.map(lambda n: _row(cfg, n), names))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def pick_account(
|
|
62
|
+
cfg: config.Config, explicit: str, hours: float, gpu: str, interactive: bool
|
|
63
|
+
) -> tuple[str, config.Profile]:
|
|
64
|
+
"""The active account runs it; kdev only steps in when it cannot finish.
|
|
65
|
+
|
|
66
|
+
There is no reason to ask while the current account has room, so the
|
|
67
|
+
question only appears when the answer matters.
|
|
68
|
+
"""
|
|
69
|
+
if explicit:
|
|
70
|
+
return explicit, cfg.profile(explicit)
|
|
71
|
+
if gpu == "none" or len(cfg.profiles) == 1:
|
|
72
|
+
# CPU sessions never touch GPU quota; one account has no alternative.
|
|
73
|
+
return cfg.active, cfg.profile(cfg.active)
|
|
74
|
+
|
|
75
|
+
with ui.spinner("checking quota…"):
|
|
76
|
+
table = rows(cfg)
|
|
77
|
+
need = hours * 3600
|
|
78
|
+
active = next((r for r in table if r.name == cfg.active), None)
|
|
79
|
+
if active and not active.error and active.gpu_left >= need:
|
|
80
|
+
return cfg.active, cfg.profile(cfg.active)
|
|
81
|
+
|
|
82
|
+
short = f"{active.hours} left" if active and not active.error else "quota unreadable"
|
|
83
|
+
roomy = [r for r in table if not r.error and r.gpu_left >= need]
|
|
84
|
+
ui.warn(f"{cfg.active} has {short}; this session needs {hours:g}h")
|
|
85
|
+
|
|
86
|
+
if not interactive:
|
|
87
|
+
if not roomy:
|
|
88
|
+
raise KdevError(
|
|
89
|
+
"No account has room for this session.",
|
|
90
|
+
"Ask for fewer --hours, or --gpu none for a CPU box.",
|
|
91
|
+
)
|
|
92
|
+
best = max(roomy, key=lambda r: r.gpu_left)
|
|
93
|
+
ui.hint(f"not a terminal, so using {best.name} ({best.hours} left)")
|
|
94
|
+
return best.name, cfg.profile(best.name)
|
|
95
|
+
|
|
96
|
+
if not roomy:
|
|
97
|
+
ui.hint(f"No account has a full {hours:g}h left; Kaggle stops the box when it runs out.")
|
|
98
|
+
name = ui.pick_profile(table, hours)
|
|
99
|
+
if name != cfg.active and ui.confirm(f"Make {name} the active account?", default=True):
|
|
100
|
+
cfg.active = name
|
|
101
|
+
config.save(cfg)
|
|
102
|
+
return name, cfg.profile(name)
|