kdev-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kdev/__init__.py +7 -0
- kdev/__main__.py +3 -0
- kdev/api.py +408 -0
- kdev/auth.py +115 -0
- kdev/bootstrap.py +510 -0
- kdev/cli.py +162 -0
- kdev/cloudflared.py +179 -0
- kdev/commands/__init__.py +5 -0
- kdev/commands/_common.py +51 -0
- kdev/commands/account.py +148 -0
- kdev/commands/box.py +694 -0
- kdev/commands/doctor.py +127 -0
- kdev/commands/home.py +92 -0
- kdev/commands/settings.py +164 -0
- kdev/commands/tunnel.py +78 -0
- kdev/commands/workspace.py +210 -0
- kdev/config.py +181 -0
- kdev/errors.py +25 -0
- kdev/kdev_box.py +538 -0
- kdev/notebook.py +261 -0
- kdev/persistence.py +314 -0
- kdev/quota.py +102 -0
- kdev/safety.py +72 -0
- kdev/session.py +284 -0
- kdev/sshcfg.py +72 -0
- kdev/tunnel.py +259 -0
- kdev/ui.py +666 -0
- kdev/wizard.py +251 -0
- kdev_cli-0.1.3.dist-info/METADATA +598 -0
- kdev_cli-0.1.3.dist-info/RECORD +33 -0
- kdev_cli-0.1.3.dist-info/WHEEL +4 -0
- kdev_cli-0.1.3.dist-info/entry_points.txt +2 -0
- kdev_cli-0.1.3.dist-info/licenses/LICENSE +21 -0
kdev/kdev_box.py
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
"""The half of kdev that runs on the Kaggle box. Standard library only.
|
|
2
|
+
|
|
3
|
+
Shipped into the container by the bootstrap cell (and refreshed by `kdev`
|
|
4
|
+
over ssh), so everything that has to happen *on* the box lives here, in one
|
|
5
|
+
module that the test suite can run locally against a temp directory.
|
|
6
|
+
|
|
7
|
+
What it is for: making `/kaggle/working` behave like a disk that is always
|
|
8
|
+
there. Kaggle commits it as the notebook's version output whenever a batch run
|
|
9
|
+
ends -- a clean exit, a crash and a cancel were all measured to commit, right
|
|
10
|
+
up to the moment the container stopped. What Kaggle does *not* keep is
|
|
11
|
+
symlinks, empty directories and executable bits, and what it cannot do is put
|
|
12
|
+
the files back. This module does both:
|
|
13
|
+
|
|
14
|
+
* `record_meta()` notes symlinks, empty directories and executable files in
|
|
15
|
+
`.kdev/meta.json`, on a timer, so a kill loses at most one interval of
|
|
16
|
+
metadata and none of the files;
|
|
17
|
+
* `restore()` rebuilds the workspace from a plan of saved versions that kdev
|
|
18
|
+
resolves on your machine, where the Kaggle login lives.
|
|
19
|
+
|
|
20
|
+
Restore is safe to run at any point in a session, for two reasons:
|
|
21
|
+
|
|
22
|
+
* a file that already existed when the restore started is never touched --
|
|
23
|
+
it was written in *this* session, so it is newer than anything saved;
|
|
24
|
+
* `.kdev/state.json` says `restored: false` until every layer is in place.
|
|
25
|
+
A session that dies half-way, or that nobody ever restored, leaves that
|
|
26
|
+
behind, and the next restore stacks this session's own saved files on top
|
|
27
|
+
of the layers it was meant to have. Nothing is skipped, so nothing done in
|
|
28
|
+
the meantime is lost.
|
|
29
|
+
|
|
30
|
+
Usage on the box:
|
|
31
|
+
kdev_box.py PLAN.json restore from a plan
|
|
32
|
+
kdev_box.py --receive read {"script", "plan"} on stdin, start a detached restore
|
|
33
|
+
kdev_box.py --watch stream restore progress until it finishes
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import contextlib
|
|
39
|
+
import json
|
|
40
|
+
import os
|
|
41
|
+
import shutil
|
|
42
|
+
import subprocess
|
|
43
|
+
import sys
|
|
44
|
+
import threading
|
|
45
|
+
import time
|
|
46
|
+
import urllib.error
|
|
47
|
+
import urllib.parse
|
|
48
|
+
import urllib.request
|
|
49
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
|
|
52
|
+
STATE = ".kdev/state.json"
|
|
53
|
+
META = ".kdev/meta.json"
|
|
54
|
+
KEYS = ".kdev/authorized_keys"
|
|
55
|
+
#: Every name a restore has put in place, one JSON string per line. Saved with
|
|
56
|
+
#: the version, so a restore that never finished still says what it delivered:
|
|
57
|
+
#: anything else under it that is gone now was deleted, not missed.
|
|
58
|
+
FETCHED = ".kdev/fetched"
|
|
59
|
+
|
|
60
|
+
#: The box's own runtime files, at the top of the workspace. Restoring them
|
|
61
|
+
#: would overwrite the live session's copies with a dead session's. Matched by
|
|
62
|
+
#: full path: a project's own web/static/custom.css is the user's file.
|
|
63
|
+
RUNTIME = {
|
|
64
|
+
"cloudflared.log",
|
|
65
|
+
".kdev-stop",
|
|
66
|
+
".kdev-session",
|
|
67
|
+
"__notebook__.ipynb",
|
|
68
|
+
"__output__.json",
|
|
69
|
+
"custom.css",
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
#: A download in progress. Kaggle saves one a killed restore left behind, and
|
|
73
|
+
#: restoring it would land on the temp path of the real file's own download.
|
|
74
|
+
PART = ".kdev-part"
|
|
75
|
+
|
|
76
|
+
WORKERS = 16
|
|
77
|
+
ATTEMPTS = 3
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def work_dir() -> Path:
|
|
81
|
+
return Path(os.environ.get("KDEV_WORK", "/kaggle/working"))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def run_dir() -> Path:
|
|
85
|
+
"""Scratch space outside /kaggle/working, so none of it is ever committed."""
|
|
86
|
+
return Path(os.environ.get("KDEV_RUN", "/root/.kdev"))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def ssh_dir() -> Path:
|
|
90
|
+
return Path(os.environ.get("KDEV_SSH_DIR", "/root/.ssh"))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _write_json(path: Path, data: dict) -> None:
|
|
94
|
+
"""Atomic, so a kill mid-write never leaves half a JSON file committed."""
|
|
95
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
97
|
+
tmp.write_text(json.dumps(data, indent=1, sort_keys=True))
|
|
98
|
+
tmp.replace(path)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _read_json(path: Path) -> dict:
|
|
102
|
+
try:
|
|
103
|
+
data = json.loads(path.read_text())
|
|
104
|
+
except (OSError, ValueError):
|
|
105
|
+
return {}
|
|
106
|
+
return data if isinstance(data, dict) else {}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def safe_rel(name: str) -> Path | None:
|
|
110
|
+
"""A saved name, or None if it would land outside the workspace."""
|
|
111
|
+
rel = Path(name)
|
|
112
|
+
if not name or rel.is_absolute() or ".." in rel.parts:
|
|
113
|
+
return None
|
|
114
|
+
return rel
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _is_kdev_path(rel: Path) -> bool:
|
|
118
|
+
return bool(rel.parts) and rel.parts[0] == ".kdev"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# --- boot ---------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def init_state(layers: list[str], session: str = "", run_by: str = "", notebook: str = "") -> None:
|
|
125
|
+
"""First thing a session does: say whether it still needs a restore.
|
|
126
|
+
|
|
127
|
+
Written before anything slow, so even a session killed during apt-get
|
|
128
|
+
records which saved versions it was supposed to be built on.
|
|
129
|
+
"""
|
|
130
|
+
_write_json(
|
|
131
|
+
work_dir() / STATE,
|
|
132
|
+
{
|
|
133
|
+
"restored": not layers,
|
|
134
|
+
"layers": list(layers),
|
|
135
|
+
"session": session,
|
|
136
|
+
"run_by": run_by,
|
|
137
|
+
# Which notebook these version labels belong to. A machine pointed at
|
|
138
|
+
# a different notebook must not resolve "v8" against its own.
|
|
139
|
+
"notebook": notebook,
|
|
140
|
+
"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
141
|
+
},
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def update_state(**fields) -> None:
|
|
146
|
+
"""Add facts to the state file as the session learns them (e.g. when it ends)."""
|
|
147
|
+
path = work_dir() / STATE
|
|
148
|
+
state = _read_json(path)
|
|
149
|
+
state.update(fields)
|
|
150
|
+
_write_json(path, state)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _key_lines(text: str) -> list[str]:
|
|
154
|
+
return [
|
|
155
|
+
ln.strip() for ln in text.splitlines() if ln.strip() and not ln.lstrip().startswith("#")
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def install_keys(*extra: str) -> list[str]:
|
|
160
|
+
"""Authorise every machine that has ever started this workspace.
|
|
161
|
+
|
|
162
|
+
The list lives in the workspace itself, so it is saved and restored with
|
|
163
|
+
everything else: a laptop that started a session last week can still reach
|
|
164
|
+
one a teammate's account started today, without anyone copying keys.
|
|
165
|
+
"""
|
|
166
|
+
persisted = work_dir() / KEYS
|
|
167
|
+
keys = _key_lines(persisted.read_text()) if persisted.exists() else []
|
|
168
|
+
for block in extra:
|
|
169
|
+
for key in _key_lines(block):
|
|
170
|
+
if key not in keys:
|
|
171
|
+
keys.append(key)
|
|
172
|
+
persisted.parent.mkdir(parents=True, exist_ok=True)
|
|
173
|
+
persisted.write_text("".join(k + "\n" for k in keys))
|
|
174
|
+
|
|
175
|
+
auth = ssh_dir() / "authorized_keys"
|
|
176
|
+
auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
177
|
+
auth.write_text("".join(k + "\n" for k in keys))
|
|
178
|
+
auth.chmod(0o600)
|
|
179
|
+
return keys
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# --- metadata Kaggle drops ----------------------------------------------------
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _restore_running() -> bool:
|
|
186
|
+
"""Whether a restore is running now.
|
|
187
|
+
|
|
188
|
+
A pid file alone is not proof: a restore killed hard leaves it behind, and
|
|
189
|
+
over a long session another process can inherit that pid -- which would
|
|
190
|
+
silently stop metadata being recorded for the rest of the session. On the
|
|
191
|
+
box, /proc says what the process actually is.
|
|
192
|
+
"""
|
|
193
|
+
try:
|
|
194
|
+
pid = int((run_dir() / "restore.pid").read_text())
|
|
195
|
+
except (OSError, ValueError):
|
|
196
|
+
return False
|
|
197
|
+
cmdline = Path(f"/proc/{pid}/cmdline")
|
|
198
|
+
if cmdline.exists():
|
|
199
|
+
try:
|
|
200
|
+
return b"kdev_box.py" in cmdline.read_bytes()
|
|
201
|
+
except OSError:
|
|
202
|
+
return False
|
|
203
|
+
try:
|
|
204
|
+
os.kill(pid, 0)
|
|
205
|
+
except OSError:
|
|
206
|
+
return False
|
|
207
|
+
return True
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def scan_meta(work: Path | None = None) -> dict:
|
|
211
|
+
"""Symlinks, empty directories and executable files under the workspace."""
|
|
212
|
+
work = work or work_dir()
|
|
213
|
+
symlinks: dict[str, str] = {}
|
|
214
|
+
dirs: list[str] = []
|
|
215
|
+
executable: list[str] = []
|
|
216
|
+
for root, dirnames, filenames in os.walk(work):
|
|
217
|
+
base = Path(root)
|
|
218
|
+
rel_root = base.relative_to(work)
|
|
219
|
+
if rel_root.parts[:1] == (".kdev",):
|
|
220
|
+
dirnames[:] = []
|
|
221
|
+
continue
|
|
222
|
+
for d in list(dirnames):
|
|
223
|
+
p = base / d
|
|
224
|
+
if p.is_symlink():
|
|
225
|
+
# os.walk lists a symlinked directory as a directory; record it
|
|
226
|
+
# as the link it is, and do not descend into its target.
|
|
227
|
+
symlinks[str(p.relative_to(work))] = os.readlink(p)
|
|
228
|
+
dirnames.remove(d)
|
|
229
|
+
if not dirnames and not filenames and rel_root.parts:
|
|
230
|
+
dirs.append(str(rel_root))
|
|
231
|
+
for f in filenames:
|
|
232
|
+
p = base / f
|
|
233
|
+
rel = str(p.relative_to(work))
|
|
234
|
+
try:
|
|
235
|
+
if p.is_symlink():
|
|
236
|
+
symlinks[rel] = os.readlink(p)
|
|
237
|
+
elif os.stat(p).st_mode & 0o111:
|
|
238
|
+
executable.append(rel)
|
|
239
|
+
except OSError:
|
|
240
|
+
continue # removed while we looked; the next pass will be right
|
|
241
|
+
return {"symlinks": symlinks, "dirs": sorted(dirs), "exec": sorted(executable)}
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def record_meta() -> bool:
|
|
245
|
+
"""Save the metadata Kaggle's snapshot drops. Called on a timer and at exit.
|
|
246
|
+
|
|
247
|
+
Skipped while a restore is running: a half-restored tree would record a
|
|
248
|
+
half-set of links, and the restore writes the merged set itself when it
|
|
249
|
+
finishes.
|
|
250
|
+
"""
|
|
251
|
+
if _restore_running():
|
|
252
|
+
return False
|
|
253
|
+
_write_json(work_dir() / META, scan_meta())
|
|
254
|
+
return True
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
# --- restore ------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _url(url: str) -> str:
|
|
261
|
+
"""Kaggle signs URLs with the file name raw -- spaces, accents and all --
|
|
262
|
+
and urllib refuses those outright, so such a file could never come back.
|
|
263
|
+
Escaping leaves what is already escaped alone."""
|
|
264
|
+
return urllib.parse.quote(url, safe=":/?#[]@!$&'()*+,;=%~")
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _fetch(url: str, dest: Path) -> None:
|
|
268
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
269
|
+
tmp = dest.with_name(dest.name + PART)
|
|
270
|
+
try:
|
|
271
|
+
with urllib.request.urlopen(_url(url), timeout=300) as r, tmp.open("wb") as out:
|
|
272
|
+
shutil.copyfileobj(r, out, 1 << 20)
|
|
273
|
+
# http.client ends a body cut off mid-stream as if it were done;
|
|
274
|
+
# a short file must not be renamed into place as a whole one.
|
|
275
|
+
if r.length:
|
|
276
|
+
raise OSError(f"connection closed {r.length} bytes short")
|
|
277
|
+
# Rename last: a half-written file must never look like a whole one.
|
|
278
|
+
tmp.replace(dest)
|
|
279
|
+
finally:
|
|
280
|
+
tmp.unlink(missing_ok=True)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _fetch_bytes(url: str) -> bytes:
|
|
284
|
+
with urllib.request.urlopen(_url(url), timeout=120) as r:
|
|
285
|
+
return r.read()
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _existing(work: Path) -> set[str]:
|
|
289
|
+
"""Everything this session already wrote, which a restore must not touch."""
|
|
290
|
+
out = set()
|
|
291
|
+
for p in work.rglob("*"):
|
|
292
|
+
rel = p.relative_to(work)
|
|
293
|
+
if _is_kdev_path(rel):
|
|
294
|
+
continue
|
|
295
|
+
if p.is_symlink() or p.is_file():
|
|
296
|
+
out.add(str(rel))
|
|
297
|
+
return out
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _merge_meta(metas: list[dict], owner: dict[str, int]) -> dict:
|
|
301
|
+
"""One metadata set from every layer, each path decided by the layer that
|
|
302
|
+
supplied it: a later regular file beats an earlier symlink, and an exec bit
|
|
303
|
+
only applies to the copy it was recorded on."""
|
|
304
|
+
symlinks: dict[str, tuple[int, str]] = {}
|
|
305
|
+
dirs: set[str] = set()
|
|
306
|
+
executable: set[str] = set()
|
|
307
|
+
for i, meta in enumerate(metas):
|
|
308
|
+
for path, target in (meta.get("symlinks") or {}).items():
|
|
309
|
+
symlinks[path] = (i, target)
|
|
310
|
+
dirs.update(meta.get("dirs") or [])
|
|
311
|
+
executable.update(p for p in meta.get("exec") or [] if owner.get(p) == i)
|
|
312
|
+
return {
|
|
313
|
+
"symlinks": {p: t for p, (i, t) in symlinks.items() if owner.get(p, -1) <= i},
|
|
314
|
+
"dirs": sorted(dirs),
|
|
315
|
+
"exec": sorted(executable),
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def apply_meta(meta: dict, preexisting: set[str], work: Path | None = None) -> list[str]:
|
|
320
|
+
"""Put back what Kaggle's snapshot dropped. Returns the paths it could not.
|
|
321
|
+
|
|
322
|
+
Each path stands alone: a name that now clashes with something else is
|
|
323
|
+
reported and skipped, never allowed to abort the rest.
|
|
324
|
+
"""
|
|
325
|
+
work = work or work_dir()
|
|
326
|
+
failed: list[str] = []
|
|
327
|
+
for d in meta.get("dirs") or []:
|
|
328
|
+
rel = safe_rel(d)
|
|
329
|
+
if rel is None:
|
|
330
|
+
continue
|
|
331
|
+
try:
|
|
332
|
+
(work / rel).mkdir(parents=True, exist_ok=True)
|
|
333
|
+
except OSError:
|
|
334
|
+
failed.append(d)
|
|
335
|
+
for name in meta.get("exec") or []:
|
|
336
|
+
rel = safe_rel(name)
|
|
337
|
+
if rel is None or name in preexisting:
|
|
338
|
+
continue
|
|
339
|
+
p = work / rel
|
|
340
|
+
try:
|
|
341
|
+
if p.is_file() and not p.is_symlink():
|
|
342
|
+
p.chmod(p.stat().st_mode | 0o111)
|
|
343
|
+
except OSError:
|
|
344
|
+
failed.append(name)
|
|
345
|
+
for name, target in (meta.get("symlinks") or {}).items():
|
|
346
|
+
rel = safe_rel(name)
|
|
347
|
+
if rel is None or name in preexisting:
|
|
348
|
+
continue
|
|
349
|
+
p = work / rel
|
|
350
|
+
try:
|
|
351
|
+
if p.is_symlink() or p.is_file():
|
|
352
|
+
p.unlink()
|
|
353
|
+
elif p.is_dir():
|
|
354
|
+
shutil.rmtree(p)
|
|
355
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
356
|
+
p.symlink_to(target)
|
|
357
|
+
except OSError:
|
|
358
|
+
failed.append(name)
|
|
359
|
+
return failed
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def restore(plan: dict, fetch=_fetch, fetch_bytes=_fetch_bytes) -> dict:
|
|
363
|
+
"""Rebuild the workspace from `plan["layers"]`, oldest first.
|
|
364
|
+
|
|
365
|
+
Each layer is {"label": "v7", "files": {name: signed_url}} exactly as
|
|
366
|
+
Kaggle's own listing returned it. Returns the final state.
|
|
367
|
+
"""
|
|
368
|
+
work = work_dir()
|
|
369
|
+
layers = plan.get("layers") or []
|
|
370
|
+
labels = [layer.get("label", "?") for layer in layers]
|
|
371
|
+
preexisting = _existing(work)
|
|
372
|
+
|
|
373
|
+
state = _read_json(work / STATE)
|
|
374
|
+
# A box already built keeps the base it was built on: filling in from
|
|
375
|
+
# another version (`kdev restore --from`) must not make the next plain
|
|
376
|
+
# `kdev restore` finish from that one instead.
|
|
377
|
+
base = state["layers"] if state.get("restored") and state.get("layers") else labels
|
|
378
|
+
state.update({"restored": False, "layers": base, "done": 0, "missing": []})
|
|
379
|
+
|
|
380
|
+
# Later layers win, so each name is fetched once, from its newest copy.
|
|
381
|
+
wanted: dict[str, tuple[int, str]] = {}
|
|
382
|
+
specials: dict[str, list[str]] = {META: [], KEYS: []}
|
|
383
|
+
for i, layer in enumerate(layers):
|
|
384
|
+
for name, url in (layer.get("files") or {}).items():
|
|
385
|
+
rel = safe_rel(name)
|
|
386
|
+
if rel is None:
|
|
387
|
+
continue
|
|
388
|
+
if name in specials:
|
|
389
|
+
specials[name].append(url)
|
|
390
|
+
continue
|
|
391
|
+
if _is_kdev_path(rel) or name in RUNTIME or name.endswith(PART) or name in preexisting:
|
|
392
|
+
continue
|
|
393
|
+
wanted[name] = (i, url)
|
|
394
|
+
state["total"] = len(wanted)
|
|
395
|
+
_write_json(work / STATE, state)
|
|
396
|
+
|
|
397
|
+
owner = {name: i for name, (i, _url) in wanted.items()}
|
|
398
|
+
metas = []
|
|
399
|
+
for url in specials[META]:
|
|
400
|
+
try:
|
|
401
|
+
metas.append(json.loads(fetch_bytes(url)))
|
|
402
|
+
except (OSError, urllib.error.URLError, ValueError):
|
|
403
|
+
metas.append({})
|
|
404
|
+
merged = _merge_meta(metas, owner)
|
|
405
|
+
# Recorded before any file arrives: Kaggle drops exec bits and links, and
|
|
406
|
+
# a restore killed part-way must still save them for the files it did
|
|
407
|
+
# bring back, or the next restore builds on copies that lost them.
|
|
408
|
+
early = scan_meta(work)
|
|
409
|
+
early["symlinks"] = {
|
|
410
|
+
**{p: t for p, t in merged["symlinks"].items() if p not in preexisting},
|
|
411
|
+
**early["symlinks"],
|
|
412
|
+
}
|
|
413
|
+
early["dirs"] = sorted({*merged["dirs"], *early["dirs"]})
|
|
414
|
+
early["exec"] = sorted({*merged["exec"], *early["exec"]})
|
|
415
|
+
_write_json(work / META, early)
|
|
416
|
+
executable = set(merged["exec"])
|
|
417
|
+
|
|
418
|
+
lock = threading.Lock()
|
|
419
|
+
last_write = [0.0]
|
|
420
|
+
delivered = (work / FETCHED).open("a")
|
|
421
|
+
|
|
422
|
+
def one(item: tuple[str, tuple[int, str]]) -> None:
|
|
423
|
+
name, (_i, url) = item
|
|
424
|
+
for attempt in range(ATTEMPTS):
|
|
425
|
+
try:
|
|
426
|
+
fetch(url, work / name)
|
|
427
|
+
# Now, not at the end: a later scan of a half-restored tree
|
|
428
|
+
# must see the bit, or it records the file as not executable.
|
|
429
|
+
if name in executable:
|
|
430
|
+
(work / name).chmod((work / name).stat().st_mode | 0o111)
|
|
431
|
+
break
|
|
432
|
+
except Exception: # per-file boundary: any failure means "missing", recorded below
|
|
433
|
+
if attempt == ATTEMPTS - 1:
|
|
434
|
+
with lock:
|
|
435
|
+
state["missing"].append(name)
|
|
436
|
+
return
|
|
437
|
+
time.sleep(2**attempt)
|
|
438
|
+
with lock:
|
|
439
|
+
state["done"] += 1
|
|
440
|
+
delivered.write(json.dumps(name) + "\n")
|
|
441
|
+
delivered.flush()
|
|
442
|
+
if time.time() - last_write[0] > 0.5:
|
|
443
|
+
last_write[0] = time.time()
|
|
444
|
+
_write_json(work / STATE, state)
|
|
445
|
+
|
|
446
|
+
with delivered, ThreadPoolExecutor(WORKERS) as pool:
|
|
447
|
+
list(pool.map(one, wanted.items()))
|
|
448
|
+
|
|
449
|
+
unapplied = apply_meta(merged, preexisting, work)
|
|
450
|
+
# Written by the restore rather than left to the timer, so a kill in the
|
|
451
|
+
# next few seconds still commits the full set -- and scanned from disk, not
|
|
452
|
+
# copied from the layers, so links and exec bits this session already made
|
|
453
|
+
# before the restore ran are recorded too.
|
|
454
|
+
_write_json(work / META, scan_meta(work))
|
|
455
|
+
|
|
456
|
+
for url in specials[KEYS]:
|
|
457
|
+
with contextlib.suppress(OSError, urllib.error.URLError, ValueError):
|
|
458
|
+
install_keys(fetch_bytes(url).decode())
|
|
459
|
+
|
|
460
|
+
if unapplied:
|
|
461
|
+
state["unapplied"] = unapplied
|
|
462
|
+
state["restored"] = True
|
|
463
|
+
state["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
464
|
+
_write_json(work / STATE, state)
|
|
465
|
+
return state
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
# --- driving it over ssh ------------------------------------------------------
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def receive(stream=sys.stdin) -> None:
|
|
472
|
+
"""Take a fresh plan (and this module's newest code) and restore, detached.
|
|
473
|
+
|
|
474
|
+
Detached so that a laptop going to sleep mid-restore does not take the
|
|
475
|
+
restore with it -- the box finishes on its own, and `--watch` can pick the
|
|
476
|
+
progress up again from any machine.
|
|
477
|
+
"""
|
|
478
|
+
payload = json.load(stream)
|
|
479
|
+
run = run_dir()
|
|
480
|
+
run.mkdir(parents=True, exist_ok=True)
|
|
481
|
+
if payload.get("script"):
|
|
482
|
+
(run / "kdev_box.py").write_text(payload["script"])
|
|
483
|
+
(run / "plan.json").write_text(json.dumps(payload["plan"]))
|
|
484
|
+
log = (run / "restore.log").open("w")
|
|
485
|
+
proc = subprocess.Popen(
|
|
486
|
+
[sys.executable, str(run / "kdev_box.py"), str(run / "plan.json")],
|
|
487
|
+
stdout=log,
|
|
488
|
+
stderr=subprocess.STDOUT,
|
|
489
|
+
stdin=subprocess.DEVNULL,
|
|
490
|
+
start_new_session=True,
|
|
491
|
+
cwd=str(run),
|
|
492
|
+
)
|
|
493
|
+
(run / "restore.pid").write_text(str(proc.pid))
|
|
494
|
+
print(json.dumps({"started": proc.pid}), flush=True)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def watch(poll: float = 1.0) -> int:
|
|
498
|
+
"""Print progress as JSON lines until the restore is done or has died."""
|
|
499
|
+
path = work_dir() / STATE
|
|
500
|
+
last = None
|
|
501
|
+
while True:
|
|
502
|
+
state = _read_json(path)
|
|
503
|
+
line = json.dumps({k: state.get(k) for k in ("restored", "done", "total", "missing")})
|
|
504
|
+
if line != last:
|
|
505
|
+
print(line, flush=True)
|
|
506
|
+
last = line
|
|
507
|
+
if state.get("restored"):
|
|
508
|
+
return 0
|
|
509
|
+
if not _restore_running():
|
|
510
|
+
# One last look: it may have finished between the two reads.
|
|
511
|
+
if _read_json(path).get("restored"):
|
|
512
|
+
continue
|
|
513
|
+
print(json.dumps({"error": "restore is not running"}), flush=True)
|
|
514
|
+
return 1
|
|
515
|
+
time.sleep(poll)
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def main(argv: list[str]) -> int:
|
|
519
|
+
if argv[:1] == ["--receive"]:
|
|
520
|
+
receive()
|
|
521
|
+
return 0
|
|
522
|
+
if argv[:1] == ["--watch"]:
|
|
523
|
+
return watch()
|
|
524
|
+
if len(argv) == 1:
|
|
525
|
+
run_dir().mkdir(parents=True, exist_ok=True)
|
|
526
|
+
(run_dir() / "restore.pid").write_text(str(os.getpid()))
|
|
527
|
+
try:
|
|
528
|
+
state = restore(json.loads(Path(argv[0]).read_text()))
|
|
529
|
+
finally:
|
|
530
|
+
(run_dir() / "restore.pid").unlink(missing_ok=True)
|
|
531
|
+
print(json.dumps({k: state.get(k) for k in ("done", "total", "missing")}))
|
|
532
|
+
return 0
|
|
533
|
+
print(__doc__, file=sys.stderr)
|
|
534
|
+
return 2
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
if __name__ == "__main__":
|
|
538
|
+
sys.exit(main(sys.argv[1:]))
|