exptrail 0.1.0__py3-none-any.whl

exptrail/__init__.py

@@ -0,0 +1,7 @@
+"""exptrail: local experiment logging that keeps README numbers verifiable."""
+
+__version__ = "0.1.0"
+
+from .run import ConfigWarning, DirtyTreeWarning, NoGitWarning, Run, track  # noqa: E402
+
+__all__ = ["Run", "track", "DirtyTreeWarning", "NoGitWarning", "ConfigWarning", "__version__"]

exptrail/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

exptrail/_stack.py

@@ -0,0 +1,27 @@
+"""Point warnings at the user's line, however many exptrail frames are in between."""
+
+from __future__ import annotations
+
+import os
+import sys
+
+_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)) + os.sep
+
+
+def _in_package(filename: str) -> bool:
+    return os.path.abspath(filename).startswith(_PACKAGE_DIR)
+
+
+def user_stacklevel() -> int:
+    """``stacklevel`` for a ``warnings.warn`` made by the caller of this function.
+
+    Walks up to the first frame outside the exptrail package, so the warning
+    names the user's ``with Run(...)``, ``run.start()`` or ``@track``-decorated
+    call, whichever path led here.
+    """
+    frame = sys._getframe(1)  # the function that is about to call warnings.warn
+    level = 1
+    while frame is not None and _in_package(frame.f_code.co_filename):
+        frame = frame.f_back
+        level += 1
+    return level

exptrail/cli.py

@@ -0,0 +1,210 @@
+"""Command-line interface: ls, show, compare, plot, table, verify, mark."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shlex
+import sys
+from pathlib import Path
+
+from .readme import render_table, verify_readme, write_table
+from .run import END_STATUSES, _now, _write_json, default_root
+from .store import SavedRun, find_run, glob_runs, list_runs
+
+
+def _fmt(value) -> str:
+    if isinstance(value, float):
+        return f"{value:.4g}"
+    if isinstance(value, (dict, list)):
+        return json.dumps(value)
+    return "—" if value is None else str(value)
+
+
+def print_table(header: list[str], rows: list[list[str]], out=None) -> None:
+    out = out or sys.stdout
+    widths = [max(len(str(c)) for c in col) for col in zip(header, *rows)]
+    for i, row in enumerate([header, *rows]):
+        print("  ".join(str(c).ljust(w) for c, w in zip(row, widths)).rstrip(), file=out)
+        if i == 0:
+            print("  ".join("-" * w for w in widths), file=out)
+
+
+def _resolve(refs: list[str], root: Path) -> list[SavedRun]:
+    return [find_run(r, root) for r in refs]
+
+
+def cmd_ls(args) -> int:
+    runs = list_runs(args.root)
+    if not runs:
+        print(f"no runs in {args.root}")
+        return 0
+    rows = []
+    for run in runs:
+        summary = run.summary
+        keys = args.metric or list(summary)[:3]
+        metrics = ", ".join(f"{k}={_fmt(summary.get(k))}" for k in keys)
+        rows.append([run.id, run.status_display(), metrics, run.short_commit()])
+    print_table(["run", "status", "summary", "commit"], rows)
+    if any(r.dirty for r in runs):
+        print("\n* = ran on a dirty working tree")
+    return 0
+
+
+def cmd_show(args) -> int:
+    run = find_run(args.run, args.root)
+    print(f"# {run.id}  ({run.path})\n")
+    print(f"status: {run.status_display()}\n")
+    for title, data in (("config", run.config), ("summary", run.summary), ("meta", run.meta)):
+        print(f"## {title}")
+        print(json.dumps(data, indent=2))
+        print()
+    return 0
+
+
+def cmd_compare(args) -> int:
+    runs = _resolve(args.runs, args.root)
+    ids = [r.id for r in runs]
+    configs = [r.config for r in runs]
+    keys = list(dict.fromkeys(k for c in configs for k in c))
+    differing = [k for k in keys if len({json.dumps(c.get(k), sort_keys=True) for c in configs}) > 1]
+    shown = keys if args.all else differing
+    print("## config" + ("" if args.all else " (differing keys only; --all for everything)"))
+    if shown:
+        print_table(["key", *ids], [[k, *(_fmt(c.get(k)) for c in configs)] for k in shown])
+    else:
+        print("(identical)")
+    summaries = [r.summary for r in runs]
+    mkeys = list(dict.fromkeys(k for s in summaries for k in s))
+    print("\n## summary")
+    print_table(["metric", *ids], [[k, *(_fmt(s.get(k)) for s in summaries)] for k in mkeys])
+    print("\n## provenance")
+    print_table(["", *ids], [["status", *(r.status_display() for r in runs)], ["commit", *(r.short_commit() for r in runs)]])
+    return 0
+
+
+def cmd_plot(args) -> int:
+    from .plot import plot_runs
+
+    runs = _resolve(args.runs, args.root)
+    out = Path(args.out or f"{args.metric}.png")
+    n = plot_runs(runs, args.metric, out)
+    if n == 0:
+        print(f"error: no run has logged {args.metric!r}", file=sys.stderr)
+        return 1
+    print(f"wrote {out} ({n} run{'s' if n != 1 else ''})")
+    return 0
+
+
+def cmd_table(args) -> int:
+    runs = []
+    for pattern in args.runs:
+        runs += [r for r in glob_runs(pattern, args.root) if r.path not in {x.path for x in runs}]
+    if not args.include_failed:
+        runs = [r for r in runs if r.status == "finished"]
+    if not runs:
+        print("error: no finished runs match " + " ".join(args.runs), file=sys.stderr)
+        return 1
+    readme = Path(args.readme)
+    command = "exptrail " + shlex.join(args.argv)
+    block = render_table(runs, args.metric, readme, args.config or [], args.precision, command,
+                         absolute_links=args.absolute_links)
+    changed = write_table(readme, block)
+    print(f"{'updated' if changed else 'unchanged'}: {readme} ({len(runs)} run{'s' if len(runs) != 1 else ''})")
+    for r in runs:
+        if r.dirty:
+            print(f"warning: {r.id} ran on a dirty working tree", file=sys.stderr)
+    return 0
+
+
+def cmd_verify(args) -> int:
+    failed = False
+    for readme in args.readme:
+        res = verify_readme(Path(readme), strict=args.strict)
+        for w in res.warnings:
+            print(f"warning: {w}", file=sys.stderr)
+        for e in res.errors:
+            print(f"MISMATCH: {e}", file=sys.stderr)
+        status = "OK" if res.ok else "FAILED"
+        print(f"{status}: {readme} — {res.rows} rows, {res.checked} values checked")
+        failed |= not res.ok
+    return 1 if failed else 0
+
+
+def cmd_mark(args) -> int:
+    run = find_run(args.run, args.root)
+    shown = run.status_display()
+    if run.status == "running" and not args.force:
+        print(f"error: {run.id} shows as {shown}; pass --force to mark it anyway", file=sys.stderr)
+        return 1
+    meta = run.read_json("meta.json")
+    previous = meta.get("status")
+    meta["status"] = args.status
+    meta["marked"] = {"by": "hand", "at": _now().isoformat(), "previous_status": previous, "shown_as": shown}
+    _write_json(run.path / "meta.json", meta)
+    print(f"marked {run.id} as {args.status} (was {shown})")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="exptrail", description=__doc__)
+    p.add_argument("--root", type=Path, default=None, help="runs directory (default: $EXPTRAIL_ROOT or ./runs)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("ls", help="list runs")
+    s.add_argument("--metric", action="append", help="summary metric to show (repeatable)")
+    s.set_defaults(func=cmd_ls)
+
+    s = sub.add_parser("show", help="show config, summary and meta of a run")
+    s.add_argument("run")
+    s.set_defaults(func=cmd_show)
+
+    s = sub.add_parser("compare", help="config diff and metric table for several runs")
+    s.add_argument("runs", nargs="+")
+    s.add_argument("--all", action="store_true", help="show all config keys, not just differing ones")
+    s.set_defaults(func=cmd_compare)
+
+    s = sub.add_parser("plot", help="overlay a metric's curves to a PNG")
+    s.add_argument("runs", nargs="+")
+    s.add_argument("--metric", required=True)
+    s.add_argument("--out", "-o", help="output PNG (default: <metric>.png)")
+    s.set_defaults(func=cmd_plot)
+
+    s = sub.add_parser("table", help="write a results table into a README")
+    s.add_argument("--metric", action="append", required=True, help="summary metric column (repeatable)")
+    s.add_argument("--runs", nargs="+", default=["*"], help="glob(s) of run folders (default: all)")
+    s.add_argument("--readme", default="README.md")
+    s.add_argument("--config", action="append", help="config key to show as a column (repeatable)")
+    s.add_argument("--precision", type=int, default=4, help="decimals for floats (default 4)")
+    s.add_argument("--include-failed", action="store_true")
+    s.add_argument("--absolute-links", action="store_true",
+                   help="link run folders by their GitHub/GitLab URL (for READMEs shown on PyPI)")
+    s.set_defaults(func=cmd_table)
+
+    s = sub.add_parser("verify", help="check README results against saved runs")
+    s.add_argument("readme", nargs="+")
+    s.add_argument("--strict", action="store_true", help="also fail on dirty or git-less runs")
+    s.set_defaults(func=cmd_verify)
+
+    s = sub.add_parser("mark", help="permanently record how a run ended (e.g. one that shows as dead)")
+    s.add_argument("run")
+    s.add_argument("status", choices=END_STATUSES)
+    s.add_argument("--force", action="store_true", help="mark it even if it still shows as running")
+    s.set_defaults(func=cmd_mark)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    args = build_parser().parse_args(argv)
+    args.root = args.root or default_root()
+    args.argv = argv
+    try:
+        return args.func(args)
+    except (LookupError, ValueError, RuntimeError, FileNotFoundError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

exptrail/liveness.py

@@ -0,0 +1,271 @@
+"""Is a run whose meta.json says ``running`` really still running?
+
+The usual ways ML jobs die (SIGKILL, the out-of-memory killer, a Colab
+disconnect, a laptop losing power) run no cleanup code, so nothing can write
+a "died" status. Instead, a run leaves evidence while it is alive:
+
+- at start, who it is: hostname, boot ID, pid and the process start time;
+- every ``heartbeat_s`` seconds, a daemon thread touches ``<run>/heartbeat``.
+
+Readers combine the two into a status at read time, without writing anything.
+On the same machine the answer is certain (``running`` or ``dead``). From
+another machine, or where the process can't be inspected, only the heartbeat's
+age is available, so an old heartbeat means ``stale``: probably dead.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import re
+import socket
+import subprocess
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+IS_WINDOWS = os.name == "nt"
+DEFAULT_HEARTBEAT_S = 30.0
+STALE_FACTOR = 3
+HEARTBEAT_FILE = "heartbeat"
+# kern.boottime (macOS) shifts when the wall clock is stepped; a real reboot
+# closer than this to the previous boot is still caught by the pid check.
+BOOT_TOLERANCE_S = 300.0
+
+
+def _positive_seconds(value, what: str) -> float:
+    try:
+        seconds = float(value)
+    except (TypeError, ValueError):
+        raise ValueError(f"{what} must be a number of seconds, got {value!r}") from None
+    if not (math.isfinite(seconds) and seconds > 0):
+        raise ValueError(f"{what} must be a positive number of seconds, got {value!r}")
+    return seconds
+
+
+def _env_seconds(name: str) -> float | None:
+    raw = os.environ.get(name, "").strip()
+    return _positive_seconds(raw, f"${name}") if raw else None
+
+
+def heartbeat_interval(value=None) -> float:
+    """``value``, else ``$EXPTRAIL_HEARTBEAT_S``, else 30 seconds."""
+    if value is not None:
+        return _positive_seconds(value, "heartbeat_s")
+    return _env_seconds("EXPTRAIL_HEARTBEAT_S") or DEFAULT_HEARTBEAT_S
+
+
+def stale_after(interval: float) -> float:
+    """``$EXPTRAIL_STALE_S``, else 3x the run's heartbeat interval."""
+    return _env_seconds("EXPTRAIL_STALE_S") or STALE_FACTOR * interval
+
+
+# -- identity of this machine and of a process ------------------------------
+
+def _run(args: list[str]) -> str:
+    try:
+        out = subprocess.run(args, capture_output=True, text=True, timeout=5,
+                             env={**os.environ, "LC_ALL": "C"})
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    return out.stdout if out.returncode == 0 else ""
+
+
+def _linux_stat(pid: int) -> list[str] | None:
+    """Fields 3.. of /proc/<pid>/stat (the command name may contain spaces)."""
+    try:
+        stat = Path(f"/proc/{pid}/stat").read_text()
+    except OSError:
+        return None
+    return stat[stat.rindex(")") + 2:].split()
+
+
+def process_start_time(pid: int) -> str | None:
+    """An opaque token that changes if ``pid`` is reused by another process."""
+    if sys.platform.startswith("linux"):
+        fields = _linux_stat(pid)
+        # field 22: start time in clock ticks since boot
+        return fields[19] if fields and len(fields) > 19 else None
+    if sys.platform == "darwin":
+        return " ".join(_run(["ps", "-o", "lstart=", "-p", str(pid)]).split()) or None
+    return None
+
+
+def _is_zombie(pid: int) -> bool:
+    if sys.platform.startswith("linux"):
+        fields = _linux_stat(pid)
+        return bool(fields) and fields[0] == "Z"
+    return False
+
+
+def boot_id() -> str | None:
+    """Changes on every reboot, so a recorded pid can't be mistaken for a new one."""
+    if sys.platform.startswith("linux"):
+        try:
+            return Path("/proc/sys/kernel/random/boot_id").read_text().strip() or None
+        except OSError:
+            return None
+    if sys.platform == "darwin":
+        m = re.search(r"sec\s*=\s*(\d+),\s*usec\s*=\s*(\d+)", _run(["sysctl", "-n", "kern.boottime"]))
+        return f"boottime:{m.group(1)}.{int(m.group(2)):06d}" if m else None
+    return None
+
+
+def _same_boot(a: str, b: str) -> bool:
+    if a.startswith("boottime:") and b.startswith("boottime:"):
+        try:
+            return abs(float(a[9:]) - float(b[9:])) < BOOT_TOLERANCE_S
+        except ValueError:
+            pass
+    return a == b
+
+
+def pid_alive(pid: int) -> bool | None:
+    """True/False, or None when it can't be checked."""
+    if IS_WINDOWS:
+        return None  # never os.kill(pid, 0) on Windows: signal 0 is CTRL_C_EVENT there
+    if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0:
+        return None  # os.kill(0 or -1, ...) would address a whole process group
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True  # exists, owned by someone else
+    except OSError:
+        return None
+    return True
+
+
+def record(heartbeat_s: float) -> dict:
+    """What a starting run writes into meta.json under ``liveness``."""
+    pid = os.getpid()
+    return {
+        "hostname": socket.gethostname(),
+        "pid": pid,
+        "proc_start": process_start_time(pid),
+        "boot_id": boot_id(),
+        "heartbeat_s": heartbeat_s,
+    }
+
+
+# -- heartbeat --------------------------------------------------------------
+
+class Heartbeat:
+    """Daemon thread that touches ``path`` every ``interval`` seconds.
+
+    Only the file's mtime is used, so it never races with meta.json writes,
+    and it beats whether or not the job calls log().
+    """
+
+    def __init__(self, path: Path, interval: float):
+        self.path = path
+        self.interval = interval
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> None:
+        self.path.touch()
+        self._thread = threading.Thread(target=self._beat, name="exptrail-heartbeat", daemon=True)
+        self._thread.start()
+
+    def _beat(self) -> None:
+        while not self._stop.wait(self.interval):
+            try:
+                os.utime(self.path)
+            except OSError:
+                pass  # folder moved or disk hiccup: keep trying, never crash the job
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread is not None and self._thread is not threading.current_thread():
+            # os.utime can block for a long time on a network or FUSE mount (Google
+            # Drive on Colab). Never hang finish() on it: the thread is a daemon, so
+            # abandoning it is safe, and it exits once the pending utime returns.
+            self._thread.join(timeout=min(self.interval, 5.0))
+        self._thread = None
+
+
+# -- read-time status -------------------------------------------------------
+
+@dataclass(frozen=True)
+class Liveness:
+    status: str  # running | dead | stale
+    reason: str = ""
+    certain: bool = False
+
+    def display(self) -> str:
+        return f"{self.status} ({self.reason})" if self.reason else self.status
+
+
+def _ago(seconds: float) -> str:
+    if seconds < 90:
+        return f"{seconds:.0f}s"
+    if seconds < 90 * 60:
+        return f"{seconds / 60:.0f}m"
+    if seconds < 48 * 3600:
+        return f"{seconds / 3600:.0f}h"
+    return f"{seconds / 86400:.0f}d"
+
+
+def _check_local(rec: dict) -> Liveness | None:
+    """Certain answer when the run was on this machine, this boot; else None."""
+    if not rec.get("hostname") or rec["hostname"] != socket.gethostname():
+        return None
+    recorded, current = rec.get("boot_id"), boot_id()
+    if not isinstance(recorded, str) or not recorded or not current:
+        return None
+    if not _same_boot(recorded, current):
+        return Liveness("dead", "machine rebooted", True)
+    pid = rec.get("pid")
+    alive = pid_alive(pid)
+    if alive is None:
+        return None
+    if not alive or _is_zombie(pid):
+        return Liveness("dead", "pid gone", True)
+    started, now_started = rec.get("proc_start"), process_start_time(pid)
+    if not started or not now_started:
+        return None
+    if str(started) != now_started:
+        return Liveness("dead", "pid reused", True)
+    return Liveness("running", "", True)
+
+
+def _last_sign_of_life(run_dir: Path) -> tuple[float, str] | None:
+    try:
+        return (run_dir / HEARTBEAT_FILE).stat().st_mtime, "heartbeat"
+    except OSError:
+        pass
+    # runs from before heartbeats existed: the newest file they wrote
+    times = []
+    for name in ("meta.json", "metrics.csv", "summary.json"):
+        try:
+            times.append((run_dir / name).stat().st_mtime)
+        except OSError:
+            pass
+    return (max(times), "activity") if times else None
+
+
+def assess(run_dir: Path, meta: dict, now: float | None = None) -> Liveness:
+    """Status of a run whose stored status is ``running``. Reads only."""
+    rec = meta.get("liveness")
+    rec = rec if isinstance(rec, dict) else {}
+    if not IS_WINDOWS:
+        local = _check_local(rec)
+        if local is not None:
+            return local
+    try:
+        interval = _positive_seconds(rec.get("heartbeat_s"), "heartbeat_s")
+    except ValueError:
+        interval = DEFAULT_HEARTBEAT_S
+    host = rec.get("hostname") or meta.get("hostname")
+    where = f"on {host}, " if host and host != socket.gethostname() else ""
+    seen = _last_sign_of_life(run_dir)
+    if seen is None:
+        return Liveness("stale", f"{where}no heartbeat")
+    last, what = seen
+    age = max(0.0, (time.time() if now is None else now) - last)
+    status = "stale" if age > stale_after(interval) else "running"
+    return Liveness(status, f"{where}last {what} {_ago(age)} ago")