kdev-cli 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kdev/kdev_box.py ADDED
@@ -0,0 +1,538 @@
1
+ """The half of kdev that runs on the Kaggle box. Standard library only.
2
+
3
+ Shipped into the container by the bootstrap cell (and refreshed by `kdev`
4
+ over ssh), so everything that has to happen *on* the box lives here, in one
5
+ module that the test suite can run locally against a temp directory.
6
+
7
+ What it is for: making `/kaggle/working` behave like a disk that is always
8
+ there. Kaggle commits it as the notebook's version output whenever a batch run
9
+ ends -- a clean exit, a crash and a cancel were all measured to commit, right
10
+ up to the moment the container stopped. What Kaggle does *not* keep is
11
+ symlinks, empty directories and executable bits, and what it cannot do is put
12
+ the files back. This module does both:
13
+
14
+ * `record_meta()` notes symlinks, empty directories and executable files in
15
+ `.kdev/meta.json`, on a timer, so a kill loses at most one interval of
16
+ metadata and none of the files;
17
+ * `restore()` rebuilds the workspace from a plan of saved versions that kdev
18
+ resolves on your machine, where the Kaggle login lives.
19
+
20
+ Restore is safe to run at any point in a session, for two reasons:
21
+
22
+ * a file that already existed when the restore started is never touched --
23
+ it was written in *this* session, so it is newer than anything saved;
24
+ * `.kdev/state.json` says `restored: false` until every layer is in place.
25
+ A session that dies half-way, or that nobody ever restored, leaves that
26
+ behind, and the next restore stacks this session's own saved files on top
27
+ of the layers it was meant to have. Nothing is skipped, so nothing done in
28
+ the meantime is lost.
29
+
30
+ Usage on the box:
31
+ kdev_box.py PLAN.json restore from a plan
32
+ kdev_box.py --receive read {"script", "plan"} on stdin, start a detached restore
33
+ kdev_box.py --watch stream restore progress until it finishes
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import contextlib
39
+ import json
40
+ import os
41
+ import shutil
42
+ import subprocess
43
+ import sys
44
+ import threading
45
+ import time
46
+ import urllib.error
47
+ import urllib.parse
48
+ import urllib.request
49
+ from concurrent.futures import ThreadPoolExecutor
50
+ from pathlib import Path
51
+
52
+ STATE = ".kdev/state.json"
53
+ META = ".kdev/meta.json"
54
+ KEYS = ".kdev/authorized_keys"
55
+ #: Every name a restore has put in place, one JSON string per line. Saved with
56
+ #: the version, so a restore that never finished still says what it delivered:
57
+ #: anything else under it that is gone now was deleted, not missed.
58
+ FETCHED = ".kdev/fetched"
59
+
60
+ #: The box's own runtime files, at the top of the workspace. Restoring them
61
+ #: would overwrite the live session's copies with a dead session's. Matched by
62
+ #: full path: a project's own web/static/custom.css is the user's file.
63
+ RUNTIME = {
64
+ "cloudflared.log",
65
+ ".kdev-stop",
66
+ ".kdev-session",
67
+ "__notebook__.ipynb",
68
+ "__output__.json",
69
+ "custom.css",
70
+ }
71
+
72
+ #: A download in progress. Kaggle saves one a killed restore left behind, and
73
+ #: restoring it would land on the temp path of the real file's own download.
74
+ PART = ".kdev-part"
75
+
76
+ WORKERS = 16
77
+ ATTEMPTS = 3
78
+
79
+
80
+ def work_dir() -> Path:
81
+ return Path(os.environ.get("KDEV_WORK", "/kaggle/working"))
82
+
83
+
84
+ def run_dir() -> Path:
85
+ """Scratch space outside /kaggle/working, so none of it is ever committed."""
86
+ return Path(os.environ.get("KDEV_RUN", "/root/.kdev"))
87
+
88
+
89
+ def ssh_dir() -> Path:
90
+ return Path(os.environ.get("KDEV_SSH_DIR", "/root/.ssh"))
91
+
92
+
93
+ def _write_json(path: Path, data: dict) -> None:
94
+ """Atomic, so a kill mid-write never leaves half a JSON file committed."""
95
+ path.parent.mkdir(parents=True, exist_ok=True)
96
+ tmp = path.with_name(path.name + ".tmp")
97
+ tmp.write_text(json.dumps(data, indent=1, sort_keys=True))
98
+ tmp.replace(path)
99
+
100
+
101
+ def _read_json(path: Path) -> dict:
102
+ try:
103
+ data = json.loads(path.read_text())
104
+ except (OSError, ValueError):
105
+ return {}
106
+ return data if isinstance(data, dict) else {}
107
+
108
+
109
+ def safe_rel(name: str) -> Path | None:
110
+ """A saved name, or None if it would land outside the workspace."""
111
+ rel = Path(name)
112
+ if not name or rel.is_absolute() or ".." in rel.parts:
113
+ return None
114
+ return rel
115
+
116
+
117
+ def _is_kdev_path(rel: Path) -> bool:
118
+ return bool(rel.parts) and rel.parts[0] == ".kdev"
119
+
120
+
121
+ # --- boot ---------------------------------------------------------------------
122
+
123
+
124
+ def init_state(layers: list[str], session: str = "", run_by: str = "", notebook: str = "") -> None:
125
+ """First thing a session does: say whether it still needs a restore.
126
+
127
+ Written before anything slow, so even a session killed during apt-get
128
+ records which saved versions it was supposed to be built on.
129
+ """
130
+ _write_json(
131
+ work_dir() / STATE,
132
+ {
133
+ "restored": not layers,
134
+ "layers": list(layers),
135
+ "session": session,
136
+ "run_by": run_by,
137
+ # Which notebook these version labels belong to. A machine pointed at
138
+ # a different notebook must not resolve "v8" against its own.
139
+ "notebook": notebook,
140
+ "started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
141
+ },
142
+ )
143
+
144
+
145
+ def update_state(**fields) -> None:
146
+ """Add facts to the state file as the session learns them (e.g. when it ends)."""
147
+ path = work_dir() / STATE
148
+ state = _read_json(path)
149
+ state.update(fields)
150
+ _write_json(path, state)
151
+
152
+
153
+ def _key_lines(text: str) -> list[str]:
154
+ return [
155
+ ln.strip() for ln in text.splitlines() if ln.strip() and not ln.lstrip().startswith("#")
156
+ ]
157
+
158
+
159
+ def install_keys(*extra: str) -> list[str]:
160
+ """Authorise every machine that has ever started this workspace.
161
+
162
+ The list lives in the workspace itself, so it is saved and restored with
163
+ everything else: a laptop that started a session last week can still reach
164
+ one a teammate's account started today, without anyone copying keys.
165
+ """
166
+ persisted = work_dir() / KEYS
167
+ keys = _key_lines(persisted.read_text()) if persisted.exists() else []
168
+ for block in extra:
169
+ for key in _key_lines(block):
170
+ if key not in keys:
171
+ keys.append(key)
172
+ persisted.parent.mkdir(parents=True, exist_ok=True)
173
+ persisted.write_text("".join(k + "\n" for k in keys))
174
+
175
+ auth = ssh_dir() / "authorized_keys"
176
+ auth.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
177
+ auth.write_text("".join(k + "\n" for k in keys))
178
+ auth.chmod(0o600)
179
+ return keys
180
+
181
+
182
+ # --- metadata Kaggle drops ----------------------------------------------------
183
+
184
+
185
+ def _restore_running() -> bool:
186
+ """Whether a restore is running now.
187
+
188
+ A pid file alone is not proof: a restore killed hard leaves it behind, and
189
+ over a long session another process can inherit that pid -- which would
190
+ silently stop metadata being recorded for the rest of the session. On the
191
+ box, /proc says what the process actually is.
192
+ """
193
+ try:
194
+ pid = int((run_dir() / "restore.pid").read_text())
195
+ except (OSError, ValueError):
196
+ return False
197
+ cmdline = Path(f"/proc/{pid}/cmdline")
198
+ if cmdline.exists():
199
+ try:
200
+ return b"kdev_box.py" in cmdline.read_bytes()
201
+ except OSError:
202
+ return False
203
+ try:
204
+ os.kill(pid, 0)
205
+ except OSError:
206
+ return False
207
+ return True
208
+
209
+
210
+ def scan_meta(work: Path | None = None) -> dict:
211
+ """Symlinks, empty directories and executable files under the workspace."""
212
+ work = work or work_dir()
213
+ symlinks: dict[str, str] = {}
214
+ dirs: list[str] = []
215
+ executable: list[str] = []
216
+ for root, dirnames, filenames in os.walk(work):
217
+ base = Path(root)
218
+ rel_root = base.relative_to(work)
219
+ if rel_root.parts[:1] == (".kdev",):
220
+ dirnames[:] = []
221
+ continue
222
+ for d in list(dirnames):
223
+ p = base / d
224
+ if p.is_symlink():
225
+ # os.walk lists a symlinked directory as a directory; record it
226
+ # as the link it is, and do not descend into its target.
227
+ symlinks[str(p.relative_to(work))] = os.readlink(p)
228
+ dirnames.remove(d)
229
+ if not dirnames and not filenames and rel_root.parts:
230
+ dirs.append(str(rel_root))
231
+ for f in filenames:
232
+ p = base / f
233
+ rel = str(p.relative_to(work))
234
+ try:
235
+ if p.is_symlink():
236
+ symlinks[rel] = os.readlink(p)
237
+ elif os.stat(p).st_mode & 0o111:
238
+ executable.append(rel)
239
+ except OSError:
240
+ continue # removed while we looked; the next pass will be right
241
+ return {"symlinks": symlinks, "dirs": sorted(dirs), "exec": sorted(executable)}
242
+
243
+
244
+ def record_meta() -> bool:
245
+ """Save the metadata Kaggle's snapshot drops. Called on a timer and at exit.
246
+
247
+ Skipped while a restore is running: a half-restored tree would record a
248
+ half-set of links, and the restore writes the merged set itself when it
249
+ finishes.
250
+ """
251
+ if _restore_running():
252
+ return False
253
+ _write_json(work_dir() / META, scan_meta())
254
+ return True
255
+
256
+
257
+ # --- restore ------------------------------------------------------------------
258
+
259
+
260
+ def _url(url: str) -> str:
261
+ """Kaggle signs URLs with the file name raw -- spaces, accents and all --
262
+ and urllib refuses those outright, so such a file could never come back.
263
+ Escaping leaves what is already escaped alone."""
264
+ return urllib.parse.quote(url, safe=":/?#[]@!$&'()*+,;=%~")
265
+
266
+
267
+ def _fetch(url: str, dest: Path) -> None:
268
+ dest.parent.mkdir(parents=True, exist_ok=True)
269
+ tmp = dest.with_name(dest.name + PART)
270
+ try:
271
+ with urllib.request.urlopen(_url(url), timeout=300) as r, tmp.open("wb") as out:
272
+ shutil.copyfileobj(r, out, 1 << 20)
273
+ # http.client ends a body cut off mid-stream as if it were done;
274
+ # a short file must not be renamed into place as a whole one.
275
+ if r.length:
276
+ raise OSError(f"connection closed {r.length} bytes short")
277
+ # Rename last: a half-written file must never look like a whole one.
278
+ tmp.replace(dest)
279
+ finally:
280
+ tmp.unlink(missing_ok=True)
281
+
282
+
283
+ def _fetch_bytes(url: str) -> bytes:
284
+ with urllib.request.urlopen(_url(url), timeout=120) as r:
285
+ return r.read()
286
+
287
+
288
+ def _existing(work: Path) -> set[str]:
289
+ """Everything this session already wrote, which a restore must not touch."""
290
+ out = set()
291
+ for p in work.rglob("*"):
292
+ rel = p.relative_to(work)
293
+ if _is_kdev_path(rel):
294
+ continue
295
+ if p.is_symlink() or p.is_file():
296
+ out.add(str(rel))
297
+ return out
298
+
299
+
300
+ def _merge_meta(metas: list[dict], owner: dict[str, int]) -> dict:
301
+ """One metadata set from every layer, each path decided by the layer that
302
+ supplied it: a later regular file beats an earlier symlink, and an exec bit
303
+ only applies to the copy it was recorded on."""
304
+ symlinks: dict[str, tuple[int, str]] = {}
305
+ dirs: set[str] = set()
306
+ executable: set[str] = set()
307
+ for i, meta in enumerate(metas):
308
+ for path, target in (meta.get("symlinks") or {}).items():
309
+ symlinks[path] = (i, target)
310
+ dirs.update(meta.get("dirs") or [])
311
+ executable.update(p for p in meta.get("exec") or [] if owner.get(p) == i)
312
+ return {
313
+ "symlinks": {p: t for p, (i, t) in symlinks.items() if owner.get(p, -1) <= i},
314
+ "dirs": sorted(dirs),
315
+ "exec": sorted(executable),
316
+ }
317
+
318
+
319
+ def apply_meta(meta: dict, preexisting: set[str], work: Path | None = None) -> list[str]:
320
+ """Put back what Kaggle's snapshot dropped. Returns the paths it could not.
321
+
322
+ Each path stands alone: a name that now clashes with something else is
323
+ reported and skipped, never allowed to abort the rest.
324
+ """
325
+ work = work or work_dir()
326
+ failed: list[str] = []
327
+ for d in meta.get("dirs") or []:
328
+ rel = safe_rel(d)
329
+ if rel is None:
330
+ continue
331
+ try:
332
+ (work / rel).mkdir(parents=True, exist_ok=True)
333
+ except OSError:
334
+ failed.append(d)
335
+ for name in meta.get("exec") or []:
336
+ rel = safe_rel(name)
337
+ if rel is None or name in preexisting:
338
+ continue
339
+ p = work / rel
340
+ try:
341
+ if p.is_file() and not p.is_symlink():
342
+ p.chmod(p.stat().st_mode | 0o111)
343
+ except OSError:
344
+ failed.append(name)
345
+ for name, target in (meta.get("symlinks") or {}).items():
346
+ rel = safe_rel(name)
347
+ if rel is None or name in preexisting:
348
+ continue
349
+ p = work / rel
350
+ try:
351
+ if p.is_symlink() or p.is_file():
352
+ p.unlink()
353
+ elif p.is_dir():
354
+ shutil.rmtree(p)
355
+ p.parent.mkdir(parents=True, exist_ok=True)
356
+ p.symlink_to(target)
357
+ except OSError:
358
+ failed.append(name)
359
+ return failed
360
+
361
+
362
+ def restore(plan: dict, fetch=_fetch, fetch_bytes=_fetch_bytes) -> dict:
363
+ """Rebuild the workspace from `plan["layers"]`, oldest first.
364
+
365
+ Each layer is {"label": "v7", "files": {name: signed_url}} exactly as
366
+ Kaggle's own listing returned it. Returns the final state.
367
+ """
368
+ work = work_dir()
369
+ layers = plan.get("layers") or []
370
+ labels = [layer.get("label", "?") for layer in layers]
371
+ preexisting = _existing(work)
372
+
373
+ state = _read_json(work / STATE)
374
+ # A box already built keeps the base it was built on: filling in from
375
+ # another version (`kdev restore --from`) must not make the next plain
376
+ # `kdev restore` finish from that one instead.
377
+ base = state["layers"] if state.get("restored") and state.get("layers") else labels
378
+ state.update({"restored": False, "layers": base, "done": 0, "missing": []})
379
+
380
+ # Later layers win, so each name is fetched once, from its newest copy.
381
+ wanted: dict[str, tuple[int, str]] = {}
382
+ specials: dict[str, list[str]] = {META: [], KEYS: []}
383
+ for i, layer in enumerate(layers):
384
+ for name, url in (layer.get("files") or {}).items():
385
+ rel = safe_rel(name)
386
+ if rel is None:
387
+ continue
388
+ if name in specials:
389
+ specials[name].append(url)
390
+ continue
391
+ if _is_kdev_path(rel) or name in RUNTIME or name.endswith(PART) or name in preexisting:
392
+ continue
393
+ wanted[name] = (i, url)
394
+ state["total"] = len(wanted)
395
+ _write_json(work / STATE, state)
396
+
397
+ owner = {name: i for name, (i, _url) in wanted.items()}
398
+ metas = []
399
+ for url in specials[META]:
400
+ try:
401
+ metas.append(json.loads(fetch_bytes(url)))
402
+ except (OSError, urllib.error.URLError, ValueError):
403
+ metas.append({})
404
+ merged = _merge_meta(metas, owner)
405
+ # Recorded before any file arrives: Kaggle drops exec bits and links, and
406
+ # a restore killed part-way must still save them for the files it did
407
+ # bring back, or the next restore builds on copies that lost them.
408
+ early = scan_meta(work)
409
+ early["symlinks"] = {
410
+ **{p: t for p, t in merged["symlinks"].items() if p not in preexisting},
411
+ **early["symlinks"],
412
+ }
413
+ early["dirs"] = sorted({*merged["dirs"], *early["dirs"]})
414
+ early["exec"] = sorted({*merged["exec"], *early["exec"]})
415
+ _write_json(work / META, early)
416
+ executable = set(merged["exec"])
417
+
418
+ lock = threading.Lock()
419
+ last_write = [0.0]
420
+ delivered = (work / FETCHED).open("a")
421
+
422
+ def one(item: tuple[str, tuple[int, str]]) -> None:
423
+ name, (_i, url) = item
424
+ for attempt in range(ATTEMPTS):
425
+ try:
426
+ fetch(url, work / name)
427
+ # Now, not at the end: a later scan of a half-restored tree
428
+ # must see the bit, or it records the file as not executable.
429
+ if name in executable:
430
+ (work / name).chmod((work / name).stat().st_mode | 0o111)
431
+ break
432
+ except Exception: # per-file boundary: any failure means "missing", recorded below
433
+ if attempt == ATTEMPTS - 1:
434
+ with lock:
435
+ state["missing"].append(name)
436
+ return
437
+ time.sleep(2**attempt)
438
+ with lock:
439
+ state["done"] += 1
440
+ delivered.write(json.dumps(name) + "\n")
441
+ delivered.flush()
442
+ if time.time() - last_write[0] > 0.5:
443
+ last_write[0] = time.time()
444
+ _write_json(work / STATE, state)
445
+
446
+ with delivered, ThreadPoolExecutor(WORKERS) as pool:
447
+ list(pool.map(one, wanted.items()))
448
+
449
+ unapplied = apply_meta(merged, preexisting, work)
450
+ # Written by the restore rather than left to the timer, so a kill in the
451
+ # next few seconds still commits the full set -- and scanned from disk, not
452
+ # copied from the layers, so links and exec bits this session already made
453
+ # before the restore ran are recorded too.
454
+ _write_json(work / META, scan_meta(work))
455
+
456
+ for url in specials[KEYS]:
457
+ with contextlib.suppress(OSError, urllib.error.URLError, ValueError):
458
+ install_keys(fetch_bytes(url).decode())
459
+
460
+ if unapplied:
461
+ state["unapplied"] = unapplied
462
+ state["restored"] = True
463
+ state["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
464
+ _write_json(work / STATE, state)
465
+ return state
466
+
467
+
468
+ # --- driving it over ssh ------------------------------------------------------
469
+
470
+
471
+ def receive(stream=sys.stdin) -> None:
472
+ """Take a fresh plan (and this module's newest code) and restore, detached.
473
+
474
+ Detached so that a laptop going to sleep mid-restore does not take the
475
+ restore with it -- the box finishes on its own, and `--watch` can pick the
476
+ progress up again from any machine.
477
+ """
478
+ payload = json.load(stream)
479
+ run = run_dir()
480
+ run.mkdir(parents=True, exist_ok=True)
481
+ if payload.get("script"):
482
+ (run / "kdev_box.py").write_text(payload["script"])
483
+ (run / "plan.json").write_text(json.dumps(payload["plan"]))
484
+ log = (run / "restore.log").open("w")
485
+ proc = subprocess.Popen(
486
+ [sys.executable, str(run / "kdev_box.py"), str(run / "plan.json")],
487
+ stdout=log,
488
+ stderr=subprocess.STDOUT,
489
+ stdin=subprocess.DEVNULL,
490
+ start_new_session=True,
491
+ cwd=str(run),
492
+ )
493
+ (run / "restore.pid").write_text(str(proc.pid))
494
+ print(json.dumps({"started": proc.pid}), flush=True)
495
+
496
+
497
+ def watch(poll: float = 1.0) -> int:
498
+ """Print progress as JSON lines until the restore is done or has died."""
499
+ path = work_dir() / STATE
500
+ last = None
501
+ while True:
502
+ state = _read_json(path)
503
+ line = json.dumps({k: state.get(k) for k in ("restored", "done", "total", "missing")})
504
+ if line != last:
505
+ print(line, flush=True)
506
+ last = line
507
+ if state.get("restored"):
508
+ return 0
509
+ if not _restore_running():
510
+ # One last look: it may have finished between the two reads.
511
+ if _read_json(path).get("restored"):
512
+ continue
513
+ print(json.dumps({"error": "restore is not running"}), flush=True)
514
+ return 1
515
+ time.sleep(poll)
516
+
517
+
518
+ def main(argv: list[str]) -> int:
519
+ if argv[:1] == ["--receive"]:
520
+ receive()
521
+ return 0
522
+ if argv[:1] == ["--watch"]:
523
+ return watch()
524
+ if len(argv) == 1:
525
+ run_dir().mkdir(parents=True, exist_ok=True)
526
+ (run_dir() / "restore.pid").write_text(str(os.getpid()))
527
+ try:
528
+ state = restore(json.loads(Path(argv[0]).read_text()))
529
+ finally:
530
+ (run_dir() / "restore.pid").unlink(missing_ok=True)
531
+ print(json.dumps({k: state.get(k) for k in ("done", "total", "missing")}))
532
+ return 0
533
+ print(__doc__, file=sys.stderr)
534
+ return 2
535
+
536
+
537
+ if __name__ == "__main__":
538
+ sys.exit(main(sys.argv[1:]))