runboard 0.2.0__py3-none-any.whl

runboard/__init__.py

@@ -0,0 +1,39 @@
+import sys
+
+from .client import Run, sync
+
+__all__ = ["init", "log", "finish", "Run", "sync", "run"]
+__version__ = "0.2.0"
+
+run = None
+_orig_excepthook = sys.excepthook
+
+
+def _excepthook(exc_type, exc, tb):
+    if run is not None and not issubclass(exc_type, KeyboardInterrupt):
+        run.finish("crashed")
+    elif run is not None:
+        run.finish("killed")
+    _orig_excepthook(exc_type, exc, tb)
+
+
+def init(project="default", name=None, config=None, **kwargs):
+    global run
+    if run is not None:
+        run.finish()
+    run = Run(project=project, name=name, config=config, **kwargs)
+    sys.excepthook = _excepthook
+    return run
+
+
+def log(data, step=None):
+    if run is None:
+        raise RuntimeError("call runboard.init() first")
+    run.log(data, step=step)
+
+
+def finish(status="finished"):
+    global run
+    if run is not None:
+        run.finish(status)
+        run = None

runboard/cli.py

@@ -0,0 +1,178 @@
+import argparse
+import getpass
+import signal
+import socket
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime
+from urllib.parse import quote
+
+from . import __version__, config
+from .client import sync
+from .server import Server
+from .storage import Storage
+
+
+def _notify(url, text):
+    try:
+        urllib.request.urlopen(urllib.request.Request(url, data=text.encode(), method="POST"), timeout=10).close()
+    except OSError as e:
+        print(f"[runboard] notify failed: {e}", file=sys.stderr)
+
+
+def _reachable_host():
+    name = socket.gethostname()
+    try:
+        socket.getaddrinfo(name, None)
+        return name
+    except OSError:
+        pass
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+            s.connect(("10.255.255.255", 1))
+            return s.getsockname()[0]
+    except OSError:
+        return "127.0.0.1"
+
+
+def cmd_serve(args):
+    signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
+    token = config.get_or_create_token()
+    try:
+        srv = Server(args.dir, token, host=args.host, port=args.port)
+    except OSError as e:
+        if args.port_explicit:
+            raise SystemExit(f"runboard: cannot bind port {args.port}: {e}")
+        srv = Server(args.dir, token, host=args.host, port=0)
+        print(f"[runboard] port {args.port} busy; using {srv.port}", file=sys.stderr)
+    srv.start()
+    advertise = args.advertise or f"http://{_reachable_host()}:{srv.port}"
+    config.write_server_info({"url": advertise, "token": token, "dir": str(srv.storage.root)})
+    print(f"runboard serving {srv.storage.root}")
+    print(f"  local:   http://127.0.0.1:{srv.port}/?token={token}")
+    print(f"  cluster: {advertise}/?token={token}")
+    print(f"  (training jobs pick up the server from {config.home() / 'server.json'})")
+    sys.stdout.flush()
+
+    tunnel = None
+
+    def open_tunnel():
+        from .tunnel import Tunnel
+
+        t = Tunnel(srv.port)
+        url = t.start()
+        public = f"{url}/?token={token}"
+        print(f"  public:  {public}", flush=True)
+        print("  (the public URL can take ~30s to start resolving)", flush=True)
+        info = config.read_server_info()
+        info["public_url"] = public
+        config.write_server_info(info)
+        if args.notify:
+            _notify(args.notify, f"runboard dashboard: {public}")
+        return t
+
+    try:
+        if args.tunnel:
+            tunnel = open_tunnel()
+        while True:
+            time.sleep(5)
+            if tunnel and tunnel.proc.poll() is not None:
+                print("[runboard] tunnel exited; restarting", file=sys.stderr, flush=True)
+                try:
+                    tunnel = open_tunnel()
+                except RuntimeError as e:
+                    print(f"[runboard] {e}; retrying in 30s", file=sys.stderr, flush=True)
+                    time.sleep(30)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if tunnel:
+            tunnel.stop()
+        srv.stop()
+
+
+def cmd_ls(args):
+    runs = Storage(args.dir).list_runs()
+    for r in runs:
+        upd = datetime.fromtimestamp(r["updated"]).strftime("%Y-%m-%d %H:%M") if r.get("updated") else "-"
+        print(f"{r['project']:20} {r['run_id']:28} {str(r.get('status', '')):10} {upd}  {r.get('name', '')}")
+
+
+def cmd_sync(args):
+    n = sync(server=args.server, token=args.token, paths=args.files or None)
+    print(f"uploaded {n} rows")
+
+
+def cmd_url(args):
+    info = config.read_server_info()
+    token = info.get("token") or config.get_or_create_token()
+    print(info.get("public_url") or f"{info.get('url', 'http://127.0.0.1:8080')}/?token={token}")
+
+
+def cmd_configure(args):
+    server = args.server.rstrip("/")
+    token = args.token or getpass.getpass("Runboard access token: ").strip()
+    if not token:
+        raise SystemExit("runboard: access token cannot be empty")
+    request = urllib.request.Request(
+        f"{server}/api/health",
+        headers={"Authorization": f"Bearer {token}", "User-Agent": "runboard/0.2"},
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=10) as response:
+            if response.status != 200:
+                raise OSError(f"HTTP {response.status}")
+    except urllib.error.HTTPError as e:
+        if e.code == 401:
+            raise SystemExit("runboard: the server rejected that token") from e
+        raise SystemExit(f"runboard: could not validate the server (HTTP {e.code})") from e
+    except OSError as e:
+        raise SystemExit(f"runboard: could not reach {server}: {e}") from e
+    config.write_server_info({"url": server, "token": token})
+    print(f"configured {server}")
+    print(f"dashboard: {server}/?token={quote(token, safe='')}")
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="runboard", description="Live training dashboards for any cluster.")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("serve", help="run the dashboard server")
+    s.add_argument("--dir", default="./runboard-runs", help="directory where runs are stored")
+    s.add_argument("--host", default="0.0.0.0")
+    s.add_argument("--port", type=int, default=None)
+    s.add_argument("--tunnel", action="store_true", help="expose publicly via a Cloudflare quick tunnel")
+    s.add_argument("--advertise", help="URL training jobs use to reach this server")
+    s.add_argument("--notify", help="POST the public URL to this webhook (e.g. https://ntfy.sh/<topic>)")
+    s.set_defaults(func=cmd_serve)
+
+    s = sub.add_parser("ls", help="list runs")
+    s.add_argument("--dir", default="./runboard-runs")
+    s.set_defaults(func=cmd_ls)
+
+    s = sub.add_parser("sync", help="upload spooled offline metrics")
+    s.add_argument("files", nargs="*")
+    s.add_argument("--server")
+    s.add_argument("--token")
+    s.set_defaults(func=cmd_sync)
+
+    s = sub.add_parser("url", help="print the dashboard URL")
+    s.set_defaults(func=cmd_url)
+
+    s = sub.add_parser("configure", help="save and verify a hosted Runboard server")
+    s.add_argument("server", help="Runboard URL, such as https://runboard.<account>.workers.dev")
+    s.add_argument("--token", help="access token; omitted values are read securely from the terminal")
+    s.set_defaults(func=cmd_configure)
+
+    p.add_argument("--version", action="version", version=f"runboard {__version__}")
+    args = p.parse_args(argv)
+    if getattr(args, "cmd", None) == "serve":
+        args.port_explicit = args.port is not None
+        args.port = args.port or 8080
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

runboard/client.py

@@ -0,0 +1,385 @@
+import atexit
+import json
+import os
+import secrets
+import socket
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from urllib.parse import quote
+
+from . import config as _config
+from .storage import Storage, clean_row, valid_name
+
+
+def _warn(msg):
+    print(f"[runboard] {msg}", file=sys.stderr, flush=True)
+
+
+def _sanitize(name):
+    s = "".join(c if c.isalnum() or c in "._-" else "-" for c in str(name))[:128].strip(".")
+    return s or "default"
+
+
+def post_json(server, token, project, run_id, payload, timeout=10):
+    url = f"{server.rstrip('/')}/api/runs/{quote(project)}/{quote(run_id)}"
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(payload, default=str).encode(),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}", "User-Agent": "runboard/0.2"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read() or b"{}")
+
+
+class _HttpSink:
+    def __init__(self, server, token):
+        self.server = server
+        self.token = token
+
+    def send(self, project, run_id, meta, rows):
+        pending = [(meta, rows)]
+        while pending:
+            batch_meta, batch = pending.pop()
+            payload = {"rows": batch}
+            if batch_meta:
+                payload["meta"] = batch_meta
+            try:
+                post_json(self.server, self.token, project, run_id, payload)
+            except urllib.error.HTTPError as e:
+                if e.code != 413 or len(batch) < 2:
+                    raise
+                middle = len(batch) // 2
+                pending.append(({}, batch[middle:]))
+                pending.append((batch_meta, batch[:middle]))
+
+
+class _FileSink:
+    def __init__(self, root):
+        self.storage = Storage(root)
+
+    def send(self, project, run_id, meta, rows):
+        if meta:
+            self.storage.update_meta(project, run_id, meta)
+        if rows:
+            self.storage.append_rows(project, run_id, rows)
+
+
+class _NullSink:
+    def send(self, project, run_id, meta, rows):
+        pass
+
+
+def _is_nonzero_rank():
+    for var in ("RANK", "LOCAL_RANK"):
+        v = os.environ.get(var)
+        if v not in (None, "", "0"):
+            return True
+    return False
+
+
+def spool_dir():
+    return _config.home() / "spool"
+
+
+class Run:
+    def __init__(
+        self,
+        project="default",
+        name=None,
+        config=None,
+        run_id=None,
+        server=None,
+        token=None,
+        dir=None,
+        mode=None,
+        tags=None,
+        flush_interval=1.0,
+        max_buffer=1_000_000,
+        chunk_size=5000,
+        all_ranks=False,
+    ):
+        self.project = _sanitize(project)
+        self.run_id = run_id or time.strftime("%Y%m%d-%H%M%S-") + secrets.token_hex(3)
+        if not valid_name(self.run_id):
+            raise ValueError(f"invalid run_id {self.run_id!r}")
+        self.name = name or self.run_id
+        self.config = dict(config or {})
+        self.flush_interval = flush_interval
+        self.max_buffer = max_buffer
+        self.chunk_size = chunk_size
+        self._send_lock = threading.Lock()
+        self._step = 0
+        self._seq = 0
+        self._sid = secrets.token_hex(8)
+        self._rows = []
+        self._overflow = []
+        self._pending_spools = []
+        self._meta = {}
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._finished = False
+        self._warned = False
+        if _is_nonzero_rank() and not all_ranks:
+            self._sink, self.mode = _NullSink(), "disabled"
+        else:
+            self._sink, self.mode = self._resolve_sink(server, token, dir, mode)
+        self._set_meta(
+            name=self.name,
+            config=self.config,
+            tags=list(tags or []),
+            status="running",
+            created=time.time(),
+            host=socket.gethostname(),
+            argv=sys.argv,
+            slurm_job_id=os.environ.get("SLURM_JOB_ID"),
+            mode=self.mode,
+        )
+        self._thread = threading.Thread(target=self._loop, name="runboard-sender", daemon=True)
+        self._thread.start()
+        atexit.register(self.finish)
+
+    def _resolve_sink(self, server, token, dir, mode):
+        dir = dir or os.environ.get("RUNBOARD_DIR")
+        if mode == "file" or (mode is None and dir):
+            root = dir or "./runboard-runs"
+            return _FileSink(root), "file"
+        info = _config.read_server_info()
+        server = server or os.environ.get("RUNBOARD_SERVER") or info.get("url")
+        token = token or os.environ.get("RUNBOARD_TOKEN") or info.get("token")
+        if server and token:
+            try:
+                urllib.request.urlopen(
+                    urllib.request.Request(
+                        f"{server.rstrip('/')}/api/health",
+                        headers={"Authorization": f"Bearer {token}", "User-Agent": "runboard/0.2"},
+                    ),
+                    timeout=3,
+                ).close()
+            except urllib.error.HTTPError as e:
+                if e.code == 401:
+                    _warn(f"server {server} rejected the token; metrics will be buffered (check RUNBOARD_TOKEN)")
+            except OSError as e:
+                _warn(f"server {server} not reachable yet ({e}); metrics will be buffered and retried")
+            return _HttpSink(server, token), "http"
+        if mode == "http":
+            raise ValueError("http mode needs a server and token (args, RUNBOARD_SERVER/RUNBOARD_TOKEN, or `runboard serve`)")
+        _warn("no server found; writing to ./runboard-runs (run `runboard serve --dir ./runboard-runs` to view)")
+        return _FileSink("./runboard-runs"), "file"
+
+    def _set_meta(self, **kw):
+        with self._lock:
+            self._meta.update(kw)
+
+    def log(self, data, step=None):
+        if self._finished:
+            return
+        if step is None:
+            step = self._step
+        self._step = max(self._step, int(step) + 1)
+        row = clean_row(dict(data))
+        row["_step"] = int(step)
+        row["_time"] = time.time()
+        with self._lock:
+            row["_seq"] = self._seq
+            row["_sid"] = self._sid
+            self._seq += 1
+            self._rows.append(row)
+            if len(self._rows) > self.max_buffer:
+                overflow = self._rows[: len(self._rows) - self.max_buffer]
+                del self._rows[: len(overflow)]
+                self._overflow.extend(overflow)
+
+    def _take(self):
+        with self._lock:
+            rows, meta = self._overflow, self._meta
+            rows.extend(self._rows)
+            self._rows, self._meta = [], {}
+            self._overflow = []
+        return meta, rows
+
+    def _restore(self, meta, rows):
+        with self._lock:
+            self._overflow[:0] = rows
+            self._meta = {**meta, **self._meta}
+
+    def _is_empty(self):
+        with self._lock:
+            return not (self._pending_spools or self._overflow or self._rows or self._meta)
+
+    def _flush_once(self):
+        with self._send_lock:
+            with self._lock:
+                pending_spool = self._pending_spools[0] if self._pending_spools else None
+            if pending_spool is not None:
+                return self._flush_spool(pending_spool)
+            meta, rows = self._take()
+            if not meta and not rows:
+                return True
+            sent = 0
+            try:
+                while True:
+                    chunk = rows[sent : sent + self.chunk_size]
+                    try:
+                        self._sink.send(self.project, self.run_id, meta, chunk)
+                    except urllib.error.HTTPError as e:
+                        if 400 <= e.code < 500 and e.code not in (401, 408, 429):
+                            _warn(f"server rejected {len(chunk)} rows ({e.code}: {e.reason}); dropping them")
+                        else:
+                            raise
+                    meta = {}
+                    sent += len(chunk)
+                    if sent >= len(rows):
+                        break
+                if self._warned:
+                    _warn("connection restored")
+                    self._warned = False
+                return True
+            except Exception as e:
+                self._restore(meta, rows[sent:])
+                if not self._warned:
+                    _warn(f"could not send metrics ({e}); buffering and retrying")
+                    self._warned = True
+                return False
+
+    def _flush_spool(self, path):
+        try:
+            rec = json.loads(path.read_text())
+            rows = rec.get("rows", [])
+            meta = rec.get("meta") or {}
+            for i in range(0, max(len(rows), 1), self.chunk_size):
+                self._sink.send(self.project, self.run_id, meta, rows[i : i + self.chunk_size])
+                meta = {}
+            path.unlink()
+            with self._lock:
+                self._pending_spools.remove(path)
+            if self._warned:
+                _warn("connection restored")
+                self._warned = False
+            return True
+        except urllib.error.HTTPError as e:
+            if 400 <= e.code < 500 and e.code not in (401, 408, 429):
+                _warn(f"server rejected spooled metrics ({e.code}: {e.reason}); dropping {path}")
+                try:
+                    path.unlink(missing_ok=True)
+                except OSError:
+                    pass
+                with self._lock:
+                    if path in self._pending_spools:
+                        self._pending_spools.remove(path)
+                return True
+            if not self._warned:
+                _warn(f"could not send metrics ({e}); buffering and retrying")
+                self._warned = True
+            return False
+        except Exception as e:
+            if not self._warned:
+                _warn(f"could not send metrics ({e}); buffering and retrying")
+                self._warned = True
+            return False
+
+    def _spill_overflow(self):
+        with self._lock:
+            rows, self._overflow = self._overflow, []
+        if not rows:
+            return
+        try:
+            paths = [self._spool({}, rows[i : i + self.chunk_size * 10]) for i in range(0, len(rows), self.chunk_size * 10)]
+        except OSError as e:
+            with self._lock:
+                self._overflow[:0] = rows
+            if not self._warned:
+                _warn(f"could not spool metrics ({e}); retaining them in memory")
+                self._warned = True
+            return
+        with self._lock:
+            self._pending_spools.extend(paths)
+
+    def _loop(self):
+        delay = self.flush_interval
+        while not self._stop.wait(delay):
+            try:
+                self._spill_overflow()
+                ok = self._flush_once()
+            except Exception:
+                ok = False
+            delay = self.flush_interval if ok else min(delay * 2, 30.0)
+
+    def _spool(self, meta, rows):
+        d = spool_dir()
+        d.mkdir(parents=True, exist_ok=True)
+        first_seq = rows[0].get("_seq", 0) if rows else 0
+        p = d / f"{self.project[:48]}__{self.run_id[:48]}__{self._sid}__{first_seq:020d}.jsonl"
+        tmp = p.with_suffix(".jsonl.tmp")
+        tmp.write_text(json.dumps({"project": self.project, "run_id": self.run_id, "meta": meta, "rows": rows}, default=str) + "\n")
+        with open(tmp, "rb") as f:
+            os.fsync(f.fileno())
+        os.replace(tmp, p)
+        fd = os.open(d, os.O_RDONLY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+        return p
+
+    def finish(self, status="finished", timeout=10.0):
+        if self._finished:
+            return
+        self._finished = True
+        self._stop.set()
+        self._thread.join(timeout=5)
+        self._set_meta(status=status, finished=time.time())
+        deadline = time.time() + timeout
+        while True:
+            if self._flush_once() and self._is_empty():
+                return
+            if time.time() >= deadline:
+                break
+            time.sleep(1.0)
+        with self._send_lock:
+            meta, rows = self._take()
+        if rows or meta:
+            try:
+                p = self._spool(meta, rows)
+                _warn(f"server unreachable; {len(rows)} rows saved to {p}. Upload later with `runboard sync`.")
+            except OSError as e:
+                self._restore(meta, rows)
+                _warn(f"server unreachable and could not write spool file ({e}); metrics remain in memory")
+        with self._lock:
+            pending_spools = list(self._pending_spools)
+        if pending_spools:
+            _warn(f"server unreachable; {len(pending_spools)} metric batches saved to {spool_dir()}. Upload later with `runboard sync`.")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.finish("crashed" if exc_type else "finished")
+
+
+def sync(server=None, token=None, paths=None):
+    info = _config.read_server_info()
+    server = server or os.environ.get("RUNBOARD_SERVER") or info.get("url")
+    token = token or os.environ.get("RUNBOARD_TOKEN") or info.get("token")
+    if not (server and token):
+        raise ValueError("no server/token configured")
+    sink = _HttpSink(server, token)
+    files = [Path(p) for p in paths] if paths else sorted(spool_dir().glob("*.jsonl"))
+    total = 0
+    for p in files:
+        for line in p.read_text().splitlines():
+            if not line.strip():
+                continue
+            rec = json.loads(line)
+            rows = rec["rows"]
+            meta = rec.get("meta") or None
+            for i in range(0, max(len(rows), 1), 5000):
+                sink.send(rec["project"], rec["run_id"], meta, rows[i : i + 5000])
+                meta = None
+            total += len(rows)
+        p.unlink()
+    return total

runboard/config.py

@@ -0,0 +1,36 @@
+import json
+import os
+import secrets
+from pathlib import Path
+
+
+def home():
+    return Path(os.environ.get("RUNBOARD_HOME", "~/.runboard")).expanduser()
+
+
+def get_or_create_token():
+    p = home() / "token"
+    if p.exists():
+        t = p.read_text().strip()
+        if t:
+            return t
+    p.parent.mkdir(parents=True, exist_ok=True)
+    t = secrets.token_urlsafe(24)
+    p.write_text(t)
+    os.chmod(p, 0o600)
+    return t
+
+
+def write_server_info(info):
+    p = home() / "server.json"
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(json.dumps(info, indent=2))
+    os.chmod(p, 0o600)
+
+
+def read_server_info():
+    p = home() / "server.json"
+    try:
+        return json.loads(p.read_text())
+    except (OSError, ValueError):
+        return {}