runboard-ml 0.2.0__py3-none-any.whl

runboard/__init__.py

@@ -0,0 +1,2 @@
+"""runboard: one self-hosted dashboard for ML training runs across projects and compute backends."""
+__version__ = "0.2.0"

runboard/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

runboard/access.py

@@ -0,0 +1,219 @@
+"""Who may use the dashboard.
+
+- This computer (a loopback connection) needs nothing: no password, no login screen.
+- Other devices (a tablet on the same Wi-Fi) only when network access is switched on, and only after pairing once:
+  this computer shows a QR code / 6-digit code, the device gets a long random key in a cookie, and only a hash of that
+  key is stored (~/.runboard/devices.json). Paired devices can be removed at any time.
+- Every request is also checked against DNS rebinding (Host header) and cross-site requests (Origin), so a web page
+  open in a browser cannot drive the dashboard.
+"""
+from __future__ import annotations
+
+import copy
+import hashlib
+import ipaddress
+import secrets
+import socket
+import threading
+import time
+from http.cookies import SimpleCookie
+
+from . import home
+
+COOKIE = "rb_device"
+PAIR_SECONDS = 120
+PAIR_TRIES = 5
+_lock = threading.Lock()
+_pairing: dict | None = None
+
+
+# ---------------------------------------------------------------- devices
+
+def _devices_file():
+    return home.HOME / "devices.json"
+
+
+_devices_cache: tuple = (None, [])
+
+
+def devices() -> list[dict]:
+    """Paired devices; every request from a tablet asks, so the file is read again only when it changed."""
+    global _devices_cache
+    f = _devices_file()
+    try:
+        st = f.stat()
+        sig = (str(f), st.st_mtime_ns, st.st_size)
+    except FileNotFoundError:
+        return []
+    if _devices_cache[0] != sig:
+        _devices_cache = (sig, home.read_json(f, []))
+    return copy.deepcopy(_devices_cache[1])
+
+
+def _hash(key: str) -> str:
+    return hashlib.sha256(key.encode()).hexdigest()
+
+
+def add_device(name: str) -> str:
+    """Creates a device and returns its cookie value (the key itself is never stored)."""
+    did, key = secrets.token_hex(4), secrets.token_urlsafe(32)
+    with _lock:
+        ds = devices()
+        ds.append({"id": did, "name": name[:60] or "device", "hash": _hash(key),
+                   "created": time.strftime("%Y-%m-%d %H:%M"), "last_seen": None})
+        home.write_json(_devices_file(), ds)
+    return f"{did}.{key}"
+
+
+def remove_device(did: str) -> bool:
+    with _lock:
+        ds = devices()
+        keep = [d for d in ds if d["id"] != did]
+        home.write_json(_devices_file(), keep)
+    return len(keep) != len(ds)
+
+
+def device_for(cookie_value: str | None) -> dict | None:
+    if not cookie_value or "." not in cookie_value:
+        return None
+    did, key = cookie_value.split(".", 1)
+    h = _hash(key)
+    for d in devices():
+        if d["id"] == did and secrets.compare_digest(d["hash"], h):
+            if not d.get("last_seen") or d["last_seen"][:16] != time.strftime("%Y-%m-%d %H:%M"):
+                with _lock:
+                    ds = devices()
+                    for x in ds:
+                        if x["id"] == did:
+                            x["last_seen"] = time.strftime("%Y-%m-%d %H:%M")
+                    home.write_json(_devices_file(), ds)
+            return d
+    return None
+
+
+def device_name(user_agent: str) -> str:
+    ua = user_agent or ""
+    for key, name in (("iPad", "iPad"), ("iPhone", "iPhone"), ("Android", "Android"), ("Macintosh", "Mac"),
+                      ("Windows", "Windows"), ("Linux", "Linux")):
+        if key in ua:
+            return name + (" tablet" if key == "Android" and "Mobile" not in ua else "")
+    return "device"
+
+
+# ---------------------------------------------------------------- pairing
+
+def start_pairing() -> dict:
+    """A new one-time pairing: a token for the QR link and a 6-digit code to type by hand. The previous one ends."""
+    global _pairing
+    with _lock:
+        _pairing = {"token": secrets.token_urlsafe(16), "code": f"{secrets.randbelow(10**6):06d}",
+                    "expires": time.time() + PAIR_SECONDS, "tries": 0}
+        return dict(_pairing)
+
+
+def redeem(token: str | None = None, code: str | None = None) -> bool:
+    """True once for the current pairing's token or code; wrong codes count, and after PAIR_TRIES it is void."""
+    global _pairing
+    with _lock:
+        p = _pairing
+        if not p or time.time() > p["expires"]:
+            _pairing = None
+            return False
+        ok = (token is not None and secrets.compare_digest(token, p["token"])) or \
+             (code is not None and secrets.compare_digest(str(code).strip(), p["code"]))
+        if ok:
+            _pairing = None
+            return True
+        p["tries"] += 1
+        if p["tries"] >= PAIR_TRIES:
+            _pairing = None
+        return False
+
+
+def pairing_state() -> dict | None:
+    p = _pairing
+    return {"seconds": max(0, round(p["expires"] - time.time()))} if p and time.time() < p["expires"] else None
+
+
+# ---------------------------------------------------------------- request checks
+
+def _local_addresses() -> set[str]:
+    out = {"127.0.0.1", "::1"}
+    try:
+        out.update(socket.gethostbyname_ex(socket.gethostname())[2])
+    except OSError:
+        pass
+    return out
+
+
+LOCAL_ADDRESSES = _local_addresses()
+
+
+def is_local_client(host: str | None) -> bool:
+    if not host:
+        return False
+    host = host.removeprefix("::ffff:")
+    return host in LOCAL_ADDRESSES or host.startswith("127.")
+
+
+def host_allowed(host_header: str, extra: list[str]) -> bool:
+    """DNS rebinding guard: a request must name this machine by IP, as localhost, by its own name or by a name listed
+    in config.yaml (allowed_hosts, e.g. a Tailscale name). A web page's own domain is never accepted."""
+    host = host_header.strip().lower()
+    if host.startswith("["):
+        host = host[1:host.find("]")]
+    elif host.count(":") == 1:
+        host = host.split(":")[0]
+    try:
+        ipaddress.ip_address(host)
+        return True
+    except ValueError:
+        pass
+    names = {"localhost", socket.gethostname().lower()} | {h.lower() for h in extra}
+    return host in names or host.endswith(".localhost")
+
+
+def cross_site(method: str, headers: dict[str, str]) -> bool:
+    """CSRF guard for anything that changes state: another site's page may not send it."""
+    if method in ("GET", "HEAD", "OPTIONS"):
+        return False
+    if headers.get("sec-fetch-site", "same-origin") not in ("same-origin", "none"):
+        return True
+    origin = headers.get("origin")
+    if origin and origin != "null":
+        return origin.split("://", 1)[-1].lower() != headers.get("host", "").lower()
+    return origin == "null"
+
+
+def cookie_value(headers: dict[str, str]) -> str | None:
+    c = SimpleCookie()
+    try:
+        c.load(headers.get("cookie", ""))
+    except Exception:
+        return None
+    return c[COOKIE].value if COOKIE in c else None
+
+
+def set_cookie_header(value: str) -> str:
+    return f"{COOKIE}={value}; Path=/; Max-Age=31536000; HttpOnly; SameSite=Strict"
+
+
+CSP = ("default-src 'self'; script-src 'self'{scripts}; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; "
+       "media-src 'self' blob:; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'")
+SECURITY_HEADERS = [
+    (b"x-frame-options", b"DENY"),
+    (b"x-content-type-options", b"nosniff"),
+    (b"referrer-policy", b"no-referrer"),
+    (b"content-security-policy", CSP.format(scripts="").encode()),
+]
+
+
+def page_csp(html: str) -> str:
+    """The policy for one page: only the page's own scripts run, named by their SHA-256. A script that got into the page
+    any other way (an injected tag or an onclick attribute) is refused by the browser."""
+    import base64
+    import re
+    hashes = [base64.b64encode(hashlib.sha256(m.encode("utf-8")).digest()).decode()
+              for m in re.findall(r"<script>(.*?)</script>", html, re.S)]
+    return CSP.format(scripts="".join(f" 'sha256-{h}'" for h in hashes))
+

runboard/backends/__init__.py

@@ -0,0 +1,12 @@
+"""Compute backends. Each one launches a run, reports its status, streams its log and brings its outputs home.
+
+Adding a backend: subclass Backend in a new module and register it in BACKENDS.
+"""
+from .base import ACTIVE, DONE, Backend
+from .kaggle import KaggleBackend
+from .local import LocalBackend
+from .remote import SlurmBackend, SSHBackend
+
+BACKENDS: dict[str, Backend] = {b.name: b for b in (KaggleBackend(), LocalBackend(), SSHBackend(), SlurmBackend())}
+
+__all__ = ["ACTIVE", "DONE", "Backend", "BACKENDS"]

runboard/backends/base.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from typing import Callable
+
+# Normalised run states shared by every backend.
+ACTIVE = {"QUEUED", "STARTING", "RUNNING", "CANCELLING"}
+DONE = {"COMPLETE", "ERROR", "CANCELLED"}
+
+
+class Backend:
+    name = "base"
+    label = "Base"
+    can_cancel = False
+    has_data = False      # can upload training data packages
+    has_quota = False
+
+    def slot(self, project) -> str | None:
+        """What a run occupies while active: one run per slot (None: no limit, the backend queues its own)."""
+        return self.name
+
+    def available(self) -> tuple[bool, str | None]:
+        """(usable, reason if not)."""
+        return True, None
+
+    def info(self) -> dict:
+        """Account / machine facts shown on the dashboard (quota, GPU …)."""
+        return {}
+
+    def submit(self, project, run_id: str, info: dict) -> dict:
+        """Launches the run; returns fields stored in the run's state (handle, url …)."""
+        raise NotImplementedError
+
+    def status(self, project, run_id: str, st: dict) -> tuple[str, str | None]:
+        """(normalised status, failure message)."""
+        raise NotImplementedError
+
+    def stream(self, project, run_id: str, st: dict, sink: list[str], alive: Callable[[], bool]) -> None:
+        """Appends live log lines to sink until the run ends (blocking)."""
+        raise NotImplementedError
+
+    def final_log(self, project, run_id: str, st: dict) -> list[str]:
+        return []
+
+    def fetch(self, project, run_id: str, st: dict) -> None:
+        """Brings outputs into the project's output directory (no-op when they are already local)."""
+
+    def cancel(self, project, run_id: str, st: dict) -> None:
+        raise NotImplementedError("This backend cannot cancel runs")
+
+    def resume_info(self, project, run_id: str, st: dict, info: dict) -> dict:
+        """What a resumed attempt needs ({resume} for the command template), or an error why it cannot resume."""
+        from ..errors import Invalid
+        cfg = (project.m.get("backends") or {}).get(self.name) or {}
+        if not cfg.get("resume"):
+            raise Invalid("no_resume", backend=self.label)
+        if st.get("status") not in ("ERROR", "CANCELLED") or st.get("backend") != self.name:
+            raise Invalid("cannot_resume", run=run_id)
+        return self._resume_info(project, run_id, st, info)
+
+    def _resume_info(self, project, run_id: str, st: dict, info: dict) -> dict:
+        raise NotImplementedError
+
+    def may_still_run(self, project, run_id: str, st: dict) -> bool:
+        """True when a run already marked as finished could in fact still be going (it is then checked again)."""
+        return False
+
+    def delete(self, project, run_id: str, st: dict) -> None:
+        """Removes the remote copy of a finished run (optional)."""
+        raise NotImplementedError

runboard/backends/kaggle.py

@@ -0,0 +1,261 @@
+"""Kaggle Notebooks: each run becomes a private single-file script kernel; data packages become private datasets.
+
+The API token stays on this machine (~/.kaggle/access_token or KAGGLE_API_TOKEN) and is never sent to the browser.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import os
+import re
+import shutil
+import zipfile
+from datetime import timezone
+from pathlib import Path
+from typing import Callable
+
+from ..errors import Invalid
+from ..home import write_json
+from ..progress import fix_text, log_lines
+from .base import Backend
+
+STATUS = {"QUEUED": "QUEUED", "RUNNING": "RUNNING", "NEW_SCRIPT": "STARTING", "COMPLETE": "COMPLETE", "ERROR": "ERROR",
+          "CANCEL_REQUESTED": "CANCELLING", "CANCEL_ACKNOWLEDGED": "CANCELLED"}
+
+
+def api():
+    import contextlib
+    import io
+
+    from kaggle.api.kaggle_api_extended import KaggleApi
+
+    a = KaggleApi()
+    # without a token, newer clients print a sign-in guide and call exit(1) instead of raising
+    with contextlib.redirect_stdout(io.StringIO()):
+        try:
+            a.authenticate()
+        except SystemExit:
+            raise RuntimeError("no Kaggle token (~/.kaggle/access_token)") from None
+    return a
+
+
+def user(a) -> str:
+    return a.get_config_value(a.CONFIG_NAME_USER)
+
+
+class KaggleBackend(Backend):
+    name = "kaggle"
+    label = "Kaggle"
+    has_data = True
+    has_quota = True
+
+    def available(self):
+        try:
+            import kaggle  # noqa: F401
+        except ImportError:
+            return False, "pip install kaggle"
+        try:
+            api()
+            return True, None
+        except Exception as e:  # missing or invalid token
+            return False, (str(e).strip().splitlines() or [type(e).__name__])[-1]
+
+    def info(self) -> dict:
+        a = api()
+        out = {"user": user(a)}
+        try:
+            q = a.quota_view()
+            g = q.gpu_quota
+            out["quota"] = {"used_h": round(g.time_used.total_seconds() / 3600, 1),
+                            "total_h": round(g.total_time_allowed.total_seconds() / 3600, 1),
+                            "refresh": q.quota_refresh_time.replace(tzinfo=timezone.utc).isoformat() if q.quota_refresh_time else None}
+        except Exception:
+            pass
+        return out
+
+    # ------------------------------------------------------------ runs
+
+    def _cfg(self, project) -> dict:
+        return (project.m.get("backends") or {}).get("kaggle") or {}
+
+    def ref(self, project, run_id: str, st: dict, a=None) -> str:
+        r = f"{user(a or api())}/{st.get('kernel') or self.slug(project, run_id)}"
+        return f"{r}/{st['version']}" if st.get("version") else r
+
+    def slug(self, project, run_id: str) -> str:
+        return re.sub(r"[^a-z0-9-]+", "-", project.render(self._cfg(project).get("kernel", project.key + "-{id|lower}"),
+                                                          {"id": run_id}).lower())
+
+    def dataset_for(self, project, info: dict) -> str | None:
+        data = project.m.get("data") or {}
+        field = self._cfg(project).get("data_field") or data.get("field")
+        if not field:
+            return None
+        value = info.get(field, project.spec(info["id"]).get(field))  # the field need not be a displayed one
+        return project.state.datasets(data.get("datasets") or {}).get(value)
+
+    def build(self, project, run_id: str, info: dict, folder: Path, owner: str) -> None:
+        cfg = self._cfg(project)
+        files = {dst: base64.b64encode((project.root / src).read_bytes()).decode()
+                 for dst, src in (cfg.get("files") or {}).items()}
+        setup = [project.render(s, info) for s in cfg.get("setup", [])]
+        command = project.render(cfg.get("resume") if info.get("resume") else cfg.get("command", ""), info)
+        code = (f"# runboard · {project.m.get('name', project.key)} · {run_id} — generated, do not edit.\n"
+                "import base64, json, os, pathlib, subprocess, sys\n"
+                f"FILES = json.loads({json.dumps(json.dumps(files))})\n"
+                "d = pathlib.Path('/tmp/runboard-code'); d.mkdir(parents=True, exist_ok=True)\n"
+                "for name, b in FILES.items():\n"
+                "    (d / name).parent.mkdir(parents=True, exist_ok=True); (d / name).write_bytes(base64.b64decode(b))\n"
+                f"for cmd in {json.dumps(setup)}:\n"
+                "    subprocess.check_call(cmd, shell=True)\n"
+                "os.environ['PYTHONUNBUFFERED'] = '1'\n"
+                f"sys.exit(subprocess.call({json.dumps(command)}, shell=True, cwd=d))\n")
+        shutil.rmtree(folder, ignore_errors=True)
+        folder.mkdir(parents=True)
+        (folder / "run.py").write_text(code, encoding="utf-8")
+        ds = self.dataset_for(project, info)
+        slug = info.get("_kernel") or self.slug(project, run_id)
+        write_json(folder / "kernel-metadata.json", {
+            "id": f"{owner}/{slug}", "title": slug, "code_file": "run.py", "language": "python", "kernel_type": "script",
+            "is_private": True, "enable_gpu": bool(cfg.get("accelerator", "NvidiaTeslaT4")), "enable_tpu": False,
+            "enable_internet": cfg.get("internet", True), "machine_shape": cfg.get("accelerator", "NvidiaTeslaT4"),
+            "dataset_sources": [f"{owner}/{ds}"] if ds else [], "competition_sources": [],
+            "kernel_sources": [f"{owner}/{info['_from_kernel']}"] if info.get("_from_kernel") else [],
+            "model_sources": []})
+
+    def submit(self, project, run_id, info):
+        data = project.m.get("data") or {}
+        field = self._cfg(project).get("data_field") or data.get("field")
+        if field and not self.dataset_for(project, info):
+            raise Invalid("not_on_kaggle", data=project.spec(run_id).get(field))
+        slug = info.get("_kernel") or self.slug(project, run_id)
+        folder = project.state.dir / "push" / slug
+        try:  # Kaggle's own wording is kept, inside a message the page can put in its language
+            a = api()
+            self.build(project, run_id, info, folder, user(a))
+            res = a.kernels_push(str(folder))
+        except Exception as e:  # noqa: BLE001
+            raise Invalid("kaggle_error", problem=(str(e).strip().splitlines() or [type(e).__name__])[-1][:300])
+        if res.error:
+            raise Invalid("kaggle_error", problem=str(res.error)[:300])
+        return {"kernel": slug, "version": res.version_number, "url": res.url}
+
+    def _resume_info(self, project, run_id, st, info):
+        """A new kernel (<slug>-r<attempt>) with the stopped kernel's output attached; Kaggle mounts it under
+        /kaggle/input/<stopped kernel>/. This works when Kaggle kept the outputs of the stopped version."""
+        prev = st.get("kernel") or self.slug(project, run_id)
+        new = f"{self.slug(project, run_id)}-r{st.get('attempt', 1) + 1}"
+        return {"resume": f"/kaggle/input/{prev}/{info['name']}", "_kernel": new, "_from_kernel": prev}
+
+    def status(self, project, run_id, st):
+        res = api().kernels_status(self.ref(project, run_id, st))
+        s = res.status.name if hasattr(res.status, "name") else str(res.status)
+        return STATUS.get(s, s), res.failure_message or None
+
+    def stream(self, project, run_id, st, sink, alive: Callable[[], bool]):
+        for ev in api().kernels_logs_stream(self.ref(project, run_id, st)):
+            if not alive():  # the run stopped or the dashboard is closing: let the thread end
+                return
+            for line in re.split(r"[\r\n]+", fix_text(ev.get("data", ""))):
+                if line.strip() and not re.search(r"Debugger warning|frozen modules|PYDEVD|debugger miss", line):
+                    sink.append(line.rstrip())
+
+    def final_log(self, project, run_id, st):
+        return log_lines(api().kernels_logs(self.ref(project, run_id, st)))
+
+    def fetch(self, project, run_id, st):
+        a = api()
+        target = project.root / self._cfg(project).get("output_dir", "runs/kaggle")
+        token = None
+        while True:
+            _, token = a.kernels_output(self.ref(project, run_id, st, a), str(target), page_token=token, force=True)
+            if not token:
+                break
+
+    def delete(self, project, run_id, st):
+        a = api()
+        a.kernels_delete(f"{user(a)}/{st.get('kernel') or self.slug(project, run_id)}", no_confirm=True)
+
+    # ------------------------------------------------------------ data
+
+    def datasets(self, project) -> list[dict]:
+        a = api()
+        owner = user(a)
+        mine = {d.ref.split("/")[1]: d for d in (a.dataset_list(mine=True) or []) if d}
+        data = project.m.get("data") or {}
+        out = []
+        for pkg, name in project.state.datasets(data.get("datasets") or {}).items():
+            d = mine.get(name)
+            try:
+                st = a.dataset_status(f"{owner}/{name}") if d else None
+            except Exception:
+                st = None
+            out.append({"key": pkg, "dataset": name, "exists": bool(d), "status": st,
+                        "version": getattr(d, "current_version_number", None),
+                        "gb": round((getattr(d, "total_bytes", 0) or 0) / 2**30, 2),
+                        "private": getattr(d, "is_private", None)})
+        return out
+
+    @staticmethod
+    def zip_folder(src: Path, dst: Path, forbid: list, say=print) -> Path:
+        files = [f for f in src.rglob("*") if f.is_file()]
+        keep = [f for f in files if not any(rx.search(f.relative_to(src).as_posix()) for rx in forbid)]
+        say(f"▶ zipping {src} ({len(keep)} files" + (f", {len(files) - len(keep)} left out by the forbidden-path rules" if len(keep) < len(files) else "") + ")…")
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        tmp = dst.with_suffix(".tmp")
+        with zipfile.ZipFile(tmp, "w", zipfile.ZIP_STORED) as z:  # images are already compressed
+            for i, f in enumerate(keep, 1):
+                z.write(f, f.relative_to(src).as_posix())
+                if i % 2000 == 0:
+                    say(f"  {i}/{len(keep)}")
+        tmp.replace(dst)
+        return dst
+
+    def upload(self, project, keys: list[str], say=print) -> int:
+        """Uploads each package to its own private dataset (new version if it exists). Runs as a background job."""
+        data = project.m.get("data") or {}
+        forbid = [re.compile(p) for p in data.get("forbid", [])]
+        catalog = {r["_key"]: r for r in project.catalog()}
+        a = api()
+        owner = user(a)
+        mine = {d.ref.split("/")[1] for d in (a.dataset_list(mine=True) or []) if d}
+        dmap = project.state.datasets(data.get("datasets") or {})
+        prefix = data.get("dataset_prefix", project.key + "-")
+        for key in keys:
+            row = catalog.get(key, {})
+            z = Path(row["_zip"]) if row else project.root / data.get("dir", ".") / f"{key}.zip"
+            if row.get("_dir"):  # a folder package: zip it now, leaving out forbidden paths (e.g. the test split)
+                z = self.zip_folder(Path(row["_dir"]), z, forbid, say)
+            if not z.exists():
+                say(f"ERROR: {z} not found")
+                return 1
+            if forbid:
+                say(f"▶ {key}: checking the archive against forbidden paths…")
+                with zipfile.ZipFile(z) as f:
+                    bad = [n for n in f.namelist() if any(rx.search(n) for rx in forbid)]
+                if bad:
+                    say(f"ERROR: {key} contains forbidden files (e.g. {bad[0]}); not uploaded")
+                    return 1
+            name = dmap.get(key) or re.sub(r"[^a-z0-9-]+", "-", f"{prefix}{key}".lower())
+            folder = project.state.dir / "upload" / name
+            shutil.rmtree(folder, ignore_errors=True)
+            folder.mkdir(parents=True)
+            try:
+                os.link(z, folder / z.name)
+            except OSError:
+                shutil.copy2(z, folder / z.name)
+            write_json(folder / "dataset-metadata.json", {"title": name, "id": f"{owner}/{name}", "licenses": [{"name": "other"}]})
+            gb = z.stat().st_size / 2**30
+            if name in mine:
+                say(f"▶ {key}: new version of {owner}/{name} ({gb:.2f} GB, private)…")
+                res = a.dataset_create_version(str(folder), f"{key} · runboard", quiet=False)
+            else:
+                say(f"▶ {key}: creating private dataset {owner}/{name} ({gb:.2f} GB)…")
+                res = a.dataset_create_new(str(folder), public=False, quiet=False)
+            shutil.rmtree(folder, ignore_errors=True)
+            if res.error:
+                say(f"ERROR: {key}: {res.error}")
+                return 1
+            project.state.set_dataset(key, name)
+            say(f"✔ {key} uploaded: {res.url} (Kaggle processes it for a few minutes)")
+        return 0