gpusched 0.3.0__py3-none-any.whl

gpusched/__init__.py

@@ -0,0 +1,13 @@
+"""gpusched — VRAM-aware single-node GPU job scheduler."""
+
+__version__ = "0.3.0"
+
+from .allocation import AllocOptions
+from .backend import NvidiaSmiBackend
+from .jobspec import JobSpec, parse_jobs_file
+from .scheduler import Scheduler, SchedulerOptions
+
+__all__ = [
+    "AllocOptions", "JobSpec", "NvidiaSmiBackend",
+    "Scheduler", "SchedulerOptions", "parse_jobs_file", "__version__",
+]

gpusched/allocation.py

@@ -0,0 +1,142 @@
+"""Pure GPU allocation logic.
+
+Separated from the scheduler loop so placement rules are testable with no
+subprocesses and no clock.
+
+Semantics (v0.2: high-water-mark aware)
+---------------------------------------
+* A job **without** a vram declaration requires a *fully idle* GPU:
+  effective external usage < idle_threshold and no scheduler job placed there.
+* A job **with** a declaration of E MiB (per GPU) can be placed on any GPU
+  whose *effective headroom* >= E + margin.
+* Effective headroom = total - effective_external - sum(own effective budgets).
+* A running job's effective budget per GPU is its declaration while its
+  observed peak stays within it; once the peak exceeds the declaration, the
+  declaration is no longer trusted and the budget escalates to
+  ``peak * (1 + spike_buffer)``. A job that has not ramped yet (actual 0)
+  still blocks its full budget — this closes the launch double-booking race.
+* ``effective_external`` is supplied by the scheduler as
+  ``max(instantaneous_external, sum(per-pid external peaks) * (1 + spike_buffer))``
+  so that fluctuating external processes are held to their observed maxima,
+  not their momentary troughs. When not supplied (unit tests / simple use),
+  it falls back to instantaneous usage minus attributed own usage.
+* A GPU hosting an *undeclared* scheduler job is never eligible for others.
+* ``exclusive=True`` additionally forbids two scheduler jobs on one GPU,
+  while still honoring headroom vs external processes.
+* Multi-GPU jobs (n_gpus=N) need N distinct GPUs, each individually
+  eligible; selection is best-fit (smallest sufficient headroom first) to
+  preserve large contiguous headroom for big jobs.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+
+from .backend import GpuSnapshot
+from .jobspec import JobSpec
+
+
+@dataclass(frozen=True)
+class Occupant:
+    """Lightweight view of a running scheduler job, for allocation purposes."""
+
+    gpu_indices: tuple[int, ...]
+    vram_mib: int | None                 # declared estimate (per GPU); None = undeclared
+    actual_mib: dict[int, int]           # last attributed usage per GPU (MiB)
+    peak_mib: dict[int, int] = field(default_factory=dict)  # observed max per GPU
+
+
+@dataclass(frozen=True)
+class AllocOptions:
+    idle_threshold_mib: int = 200   # GPU counts as idle below this usage
+    margin_mib: int = 512           # safety margin on top of declarations
+    exclusive: bool = False         # one scheduler job per GPU
+    spike_buffer: float = 0.10      # headroom buffer over observed maxima
+
+
+def effective_budget(occ: Occupant, gpu: int, spike_buffer: float) -> int | None:
+    """MiB to hold against `occ` on `gpu`. None = the whole device (undeclared)."""
+    if occ.vram_mib is None:
+        return None
+    peak = occ.peak_mib.get(gpu, 0)
+    if peak <= occ.vram_mib:
+        budget = occ.vram_mib
+    else:  # declaration violated -> trust the empirical max, buffered
+        budget = math.ceil(round(peak * (1 + spike_buffer), 6))
+    return max(budget, occ.actual_mib.get(gpu, 0))
+
+
+def _fallback_external(gpu: int, snapshot: GpuSnapshot, occupants: list[Occupant]) -> int:
+    own_actual = sum(occ.actual_mib.get(gpu, 0) for occ in occupants if gpu in occ.gpu_indices)
+    return max(0, snapshot.gpus[gpu].used_mib - own_actual)
+
+
+def effective_headroom(
+    gpu: int,
+    snapshot: GpuSnapshot,
+    occupants: list[Occupant],
+    opts: AllocOptions,
+    external_mib: dict[int, int] | None = None,
+) -> int | None:
+    """Headroom available for a *new declared* job on `gpu`.
+
+    Returns None if the GPU is off-limits (hosts an undeclared job).
+    """
+    stat = snapshot.gpus[gpu]
+    own = 0
+    for occ in occupants:
+        if gpu not in occ.gpu_indices:
+            continue
+        budget = effective_budget(occ, gpu, opts.spike_buffer)
+        if budget is None:
+            return None  # undeclared job owns the whole device
+        own += budget
+    external = (
+        external_mib[gpu] if external_mib is not None
+        else _fallback_external(gpu, snapshot, occupants)
+    )
+    return stat.total_mib - external - own
+
+
+def find_allocation(
+    spec: JobSpec,
+    snapshot: GpuSnapshot,
+    occupants: list[Occupant],
+    allowed_gpus: set[int],
+    opts: AllocOptions,
+    external_mib: dict[int, int] | None = None,
+) -> list[int] | None:
+    """Return the GPU indices to run `spec` on, or None if it cannot start now."""
+    candidates = sorted(g for g in snapshot.gpus if g in allowed_gpus)
+    scheduler_used = {g for occ in occupants for g in occ.gpu_indices}
+
+    def external(g: int) -> int:
+        return (
+            external_mib[g] if external_mib is not None
+            else _fallback_external(g, snapshot, occupants)
+        )
+
+    if spec.vram_mib is None:
+        # Undeclared: fully idle GPUs only (judged on EFFECTIVE external usage,
+        # so a live external process is held to its peak), lowest index first.
+        eligible = [
+            g for g in candidates
+            if g not in scheduler_used and external(g) < opts.idle_threshold_mib
+        ]
+        return eligible[: spec.n_gpus] if len(eligible) >= spec.n_gpus else None
+
+    need = spec.vram_mib + opts.margin_mib
+    scored: list[tuple[int, int]] = []  # (headroom, gpu)
+    for g in candidates:
+        if opts.exclusive and g in scheduler_used:
+            continue
+        headroom = effective_headroom(g, snapshot, occupants, opts, external_mib)
+        if headroom is not None and headroom >= need:
+            scored.append((headroom, g))
+
+    if len(scored) < spec.n_gpus:
+        return None
+    # Best-fit: smallest sufficient headroom first; tie-break on index.
+    scored.sort()
+    return sorted(g for _, g in scored[: spec.n_gpus])

gpusched/backend.py

@@ -0,0 +1,130 @@
+"""GPU state backends.
+
+The scheduler only ever sees a :class:`GpuSnapshot`; how it is produced is
+behind the :class:`GpuBackend` protocol. The real backend shells out to
+``nvidia-smi``; test backends fabricate snapshots (see ``gpusched.testing``).
+"""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass, field
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class GpuStat:
+    index: int
+    total_mib: int
+    used_mib: int
+    util_pct: int | None = None     # device-level SM utilization, if known
+
+    @property
+    def free_mib(self) -> int:
+        return self.total_mib - self.used_mib
+
+
+@dataclass(frozen=True)
+class ProcStat:
+    gpu_index: int
+    pid: int
+    used_mib: int
+
+
+@dataclass
+class GpuSnapshot:
+    gpus: dict[int, GpuStat] = field(default_factory=dict)
+    procs: list[ProcStat] = field(default_factory=list)
+
+
+class GpuBackend(Protocol):
+    def snapshot(self) -> GpuSnapshot: ...
+
+
+class BackendError(RuntimeError):
+    pass
+
+
+def pgid_of(pid: int) -> int | None:
+    """Process-group id of *pid* via /proc (Linux). None if the process is gone.
+
+    Used to attribute GPU compute processes to scheduler-launched jobs: each
+    job is started in its own session (``os.setsid``), so every descendant —
+    including those spawned through ``sh -c`` — shares the job's pgid.
+    """
+    try:
+        with open(f"/proc/{pid}/stat", "rb") as f:
+            data = f.read().decode("utf-8", "replace")
+        # /proc/[pid]/stat: "pid (comm) state ppid pgrp session ...".
+        # comm may itself contain spaces/parens, so split after the LAST ')':
+        after_comm = data.rsplit(")", 1)[1].split()
+        return int(after_comm[2])  # [0]=state, [1]=ppid, [2]=pgrp
+    except (FileNotFoundError, ProcessLookupError, PermissionError, IndexError, ValueError):
+        return None
+
+
+class NvidiaSmiBackend:
+    """Real backend: two nvidia-smi queries per snapshot.
+
+    * ``--query-gpu=index,uuid,memory.total,memory.used`` for GPU-level stats
+    * ``--query-compute-apps=gpu_uuid,pid,used_memory`` for per-process stats
+    """
+
+    def __init__(self, nvidia_smi: str = "nvidia-smi", timeout: float = 10.0):
+        self._bin = nvidia_smi
+        self._timeout = timeout
+
+    def _run(self, args: list[str]) -> str:
+        try:
+            return subprocess.check_output(
+                [self._bin, *args], text=True, timeout=self._timeout,
+                stderr=subprocess.PIPE,
+            )
+        except FileNotFoundError as e:
+            raise BackendError(f"{self._bin} not found — is the NVIDIA driver installed?") from e
+        except subprocess.CalledProcessError as e:
+            raise BackendError(f"{self._bin} failed: {e.stderr or e}") from e
+        except subprocess.TimeoutExpired as e:
+            raise BackendError(f"{self._bin} timed out after {self._timeout}s") from e
+
+    def snapshot(self) -> GpuSnapshot:
+        snap = GpuSnapshot()
+        uuid_to_index: dict[str, int] = {}
+
+        out = self._run([
+            "--query-gpu=index,uuid,memory.total,memory.used,utilization.gpu",
+            "--format=csv,noheader,nounits",
+        ])
+        for line in out.strip().splitlines():
+            parts = [p.strip() for p in line.split(",")]
+            if len(parts) != 5:
+                continue
+            idx, uuid, total, used, util = parts
+            try:
+                util_pct: int | None = int(util)
+            except ValueError:
+                util_pct = None  # "[N/A]" on some virtualized setups
+            gpu = GpuStat(index=int(idx), total_mib=int(total),
+                          used_mib=int(used), util_pct=util_pct)
+            snap.gpus[gpu.index] = gpu
+            uuid_to_index[uuid] = gpu.index
+
+        out = self._run([
+            "--query-compute-apps=gpu_uuid,pid,used_memory",
+            "--format=csv,noheader,nounits",
+        ])
+        for line in out.strip().splitlines():
+            parts = [p.strip() for p in line.split(",")]
+            if len(parts) != 3 or parts[0] not in uuid_to_index:
+                continue
+            try:
+                snap.procs.append(ProcStat(
+                    gpu_index=uuid_to_index[parts[0]],
+                    pid=int(parts[1]),
+                    used_mib=int(parts[2]),
+                ))
+            except ValueError:
+                # used_memory can be "[N/A]" (e.g. inside some containers /
+                # for graphics processes); skip — attribution degrades to n/a.
+                continue
+        return snap

gpusched/cli.py

@@ -0,0 +1,112 @@
+"""Command-line interface: ``gpusched jobs.txt [options]``."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__
+from .allocation import AllocOptions
+from .backend import NvidiaSmiBackend
+from .jobspec import JobSpecError, parse_jobs_file
+from .scheduler import Scheduler, SchedulerOptions
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="gpusched",
+        description=(
+            "VRAM-aware GPU job scheduler. Reads one shell command per line; "
+            "lines may declare expected max VRAM: '[vram=18G] python train.py' "
+            "or multiple GPUs: '[vram=30G gpus=2] torchrun ...'. Declared jobs "
+            "are packed onto GPUs with enough free memory; undeclared jobs get "
+            "a fully idle GPU. Actual per-job VRAM is monitored and compared "
+            "against declarations."
+        ),
+    )
+    p.add_argument("jobs_file", help="file with one shell command per line")
+    p.add_argument("--gpus", default=None, metavar="0,1,3",
+                   help="comma-separated GPU indices to use (default: all visible)")
+    p.add_argument("--idle-threshold", type=int, default=200, metavar="MIB",
+                   help="GPU counts as idle below this usage, for undeclared jobs (default: 200)")
+    p.add_argument("--margin", type=int, default=512, metavar="MIB",
+                   help="safety margin added to every vram declaration (default: 512)")
+    p.add_argument("--tolerance", type=float, default=0.10, metavar="FRAC",
+                   help="relative band before flagging over/under-declaration (default: 0.10)")
+    p.add_argument("--poll", type=float, default=5.0, metavar="SEC",
+                   help="polling interval in seconds (default: 5)")
+    p.add_argument("--spike-buffer", type=float, default=0.10, metavar="FRAC",
+                   help="headroom buffer applied over observed VRAM maxima of fluctuating "
+                        "processes — both external ones and own jobs that exceeded their "
+                        "declaration (default: 0.10)")
+    p.add_argument("--exclusive", action="store_true",
+                   help="never co-locate two scheduler jobs on one GPU, even if declarations fit")
+    p.add_argument("--log-dir", default="gpusched_logs",
+                   help="directory for per-job stdout/stderr logs (default: gpusched_logs)")
+    p.add_argument("--watch", action="store_true",
+                   help="keep running after the queue drains, picking up lines appended "
+                        "to the jobs file (the jobs file is re-read every poll either way)")
+    p.add_argument("--oom-retries", type=int, default=0, metavar="N",
+                   help="default CUDA-OOM auto-retries per job; [retries=N] overrides (default: 0)")
+    p.add_argument("--fresh", action="store_true",
+                   help="ignore and remove the existing journal: re-run everything")
+    p.add_argument("-v", "--verbose", action="store_true",
+                   help="stream live per-job VRAM usage as peaks grow")
+    p.add_argument("--sim", type=int, default=None, metavar="N_GPUS",
+                   help="dry-run against N simulated 24 GiB GPUs (no hardware needed); "
+                        "pair with 'python -m gpusched.simjob' commands")
+    p.add_argument("--version", action="version", version=f"gpusched {__version__}")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    try:
+        jobs = parse_jobs_file(args.jobs_file)
+    except JobSpecError as e:
+        print(f"gpusched: {e}", file=sys.stderr)
+        return 2
+    except OSError as e:
+        print(f"gpusched: cannot read jobs file: {e}", file=sys.stderr)
+        return 2
+    if not jobs and not args.watch:
+        print("gpusched: jobs file contains no jobs (use --watch to wait for some)", file=sys.stderr)
+        return 2
+
+    if args.fresh:
+        import pathlib
+        pathlib.Path(args.log_dir, "journal.jsonl").unlink(missing_ok=True)
+
+    if args.sim is not None:
+        from .testing import SimBackend
+        backend = SimBackend(n_gpus=args.sim)
+    else:
+        backend = NvidiaSmiBackend()
+
+    allowed = (
+        {int(g) for g in args.gpus.split(",")} if args.gpus else None
+    )
+    options = SchedulerOptions(
+        alloc=AllocOptions(
+            idle_threshold_mib=args.idle_threshold,
+            margin_mib=args.margin,
+            exclusive=args.exclusive,
+            spike_buffer=args.spike_buffer,
+        ),
+        poll_interval=args.poll,
+        tolerance=args.tolerance,
+        verbose=args.verbose,
+        log_dir=args.log_dir,
+        watch=args.watch,
+        oom_retries_default=args.oom_retries,
+    )
+    sched = Scheduler(backend, jobs_path=args.jobs_file, allowed_gpus=allowed, options=options)
+    try:
+        return sched.run()
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

gpusched/jobspec.py

@@ -0,0 +1,161 @@
+"""Job specification parsing.
+
+Jobs-file syntax (one job per line):
+
+    # comment / blank lines are skipped
+    python train.py --config a.yaml          # no estimate -> needs an idle GPU
+    [vram=18000] python train.py             # declares max 18000 MiB
+    [vram=22G] bash run_eval.sh              # G / GiB suffix accepted
+    [vram=30G gpus=2] torchrun train_big.py  # 2 GPUs, each with >= 30 GiB free
+    [timeout=2h] python flaky_eval.py        # SIGTERM after 2 hours (opt-in only)
+    [vram=8G retries=2] python sweep.py      # auto-retry on CUDA OOM, up to 2x
+
+Attribute block must be a single leading ``[key=value ...]`` group.
+``vram`` is interpreted **per GPU** for multi-GPU jobs. ``timeout`` accepts
+s/m/h/d suffixes (default seconds). Jobs are identified by a hash of their
+command text (plus an occurrence counter for duplicate lines), which is what
+makes live-edited queue files and resume-after-restart well defined.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+_ATTR_BLOCK = re.compile(r"^\[(?P<attrs>[^\]]*)\]\s*(?P<cmd>.*)$", re.DOTALL)
+_VRAM_VALUE = re.compile(r"^(?P<num>\d+(?:\.\d+)?)\s*(?P<unit>g|gb|gib|m|mb|mib)?$", re.IGNORECASE)
+_DURATION = re.compile(r"^(?P<num>\d+(?:\.\d+)?)\s*(?P<unit>s|m|h|d)?$", re.IGNORECASE)
+
+
+class JobSpecError(ValueError):
+    """Raised on malformed jobs-file lines; carries the 1-based line number."""
+
+    def __init__(self, lineno: int, message: str):
+        super().__init__(f"jobs file line {lineno}: {message}")
+        self.lineno = lineno
+
+
+@dataclass(frozen=True)
+class JobSpec:
+    """A single queued job."""
+
+    index: int                      # 1-based position in the jobs file
+    command: str                    # shell command to execute
+    vram_mib: int | None = None     # declared max VRAM per GPU (MiB); None = undeclared
+    n_gpus: int = 1                 # number of GPUs required
+    timeout_s: float | None = None  # walltime before SIGTERM; None = never (opt-in)
+    retries: int = 0                # auto-retries on CUDA OOM
+    key: str = field(default="", compare=False)   # stable identity (hash#occurrence)
+    lineno: int = field(default=0, compare=False)
+
+    @property
+    def label(self) -> str:
+        return f"job {self.index}"
+
+
+def parse_vram(value: str, lineno: int = 0) -> int:
+    """Parse a vram value like '12000', '12.5G', '24GiB', '8000MiB' -> MiB."""
+    m = _VRAM_VALUE.match(value.strip())
+    if not m:
+        raise JobSpecError(lineno, f"cannot parse vram value {value!r} (use MiB or a G/GiB suffix)")
+    num = float(m.group("num"))
+    unit = (m.group("unit") or "m").lower()
+    mib = num * 1024 if unit.startswith("g") else num
+    mib_int = int(round(mib))
+    if mib_int <= 0:
+        raise JobSpecError(lineno, f"vram must be positive, got {value!r}")
+    return mib_int
+
+
+def parse_duration(value: str, lineno: int = 0) -> float:
+    """Parse '90', '90s', '15m', '2h', '1.5d' -> seconds."""
+    m = _DURATION.match(value.strip())
+    if not m:
+        raise JobSpecError(lineno, f"cannot parse duration {value!r} (use s/m/h/d)")
+    mult = {"s": 1, "m": 60, "h": 3600, "d": 86400}[(m.group("unit") or "s").lower()]
+    sec = float(m.group("num")) * mult
+    if sec <= 0:
+        raise JobSpecError(lineno, f"timeout must be positive, got {value!r}")
+    return sec
+
+
+def job_key(command: str, occurrence: int) -> str:
+    """Stable identity for a command line; occurrence disambiguates duplicates."""
+    import hashlib
+    h = hashlib.sha1(command.strip().encode()).hexdigest()[:12]
+    return f"{h}#{occurrence}"
+
+
+def parse_line(line: str, lineno: int, index: int) -> JobSpec | None:
+    """Parse one jobs-file line. Returns None for blanks/comments."""
+    stripped = line.strip()
+    if not stripped or stripped.startswith("#"):
+        return None
+
+    vram_mib: int | None = None
+    n_gpus = 1
+    timeout_s: float | None = None
+    retries = 0
+    command = stripped
+
+    if stripped.startswith("[") and not _ATTR_BLOCK.match(stripped):
+        # A line beginning with '[' but never closed is almost certainly a
+        # torn mid-edit attribute block — refuse to execute it as shell.
+        raise JobSpecError(lineno, "unterminated '[...]' attribute block")
+
+    m = _ATTR_BLOCK.match(stripped)
+    if m:
+        command = m.group("cmd").strip()
+        if not command:
+            raise JobSpecError(lineno, "attribute block present but command is empty")
+        for token in m.group("attrs").split():
+            if "=" not in token:
+                raise JobSpecError(lineno, f"malformed attribute {token!r} (expected key=value)")
+            key, _, value = token.partition("=")
+            key = key.lower()
+            if key == "vram":
+                vram_mib = parse_vram(value, lineno)
+            elif key == "timeout":
+                timeout_s = parse_duration(value, lineno)
+            elif key == "retries":
+                try:
+                    retries = int(value)
+                except ValueError:
+                    raise JobSpecError(lineno, f"retries must be an integer, got {value!r}") from None
+                if retries < 0:
+                    raise JobSpecError(lineno, f"retries must be >= 0, got {retries}")
+            elif key == "gpus":
+                try:
+                    n_gpus = int(value)
+                except ValueError:
+                    raise JobSpecError(lineno, f"gpus must be an integer, got {value!r}") from None
+                if n_gpus < 1:
+                    raise JobSpecError(lineno, f"gpus must be >= 1, got {n_gpus}")
+            else:
+                raise JobSpecError(lineno, f"unknown attribute {key!r} (known: vram, gpus, timeout, retries)")
+
+    return JobSpec(index=index, command=command, vram_mib=vram_mib, n_gpus=n_gpus,
+                   timeout_s=timeout_s, retries=retries, lineno=lineno)
+
+
+def assign_keys(specs: list[JobSpec]) -> list[JobSpec]:
+    """Attach stable identity keys (command hash + occurrence index)."""
+    from dataclasses import replace
+    seen: dict[str, int] = {}
+    out = []
+    for spec in specs:
+        occ = seen.get(spec.command, 0)
+        seen[spec.command] = occ + 1
+        out.append(replace(spec, key=job_key(spec.command, occ)))
+    return out
+
+
+def parse_jobs_file(path: str) -> list[JobSpec]:
+    """Parse a jobs file into an ordered list of JobSpecs with identity keys."""
+    specs: list[JobSpec] = []
+    with open(path, encoding="utf-8") as f:
+        for lineno, line in enumerate(f, start=1):
+            spec = parse_line(line, lineno, index=len(specs) + 1)
+            if spec is not None:
+                specs.append(spec)
+    return assign_keys(specs)

gpusched/journal.py

@@ -0,0 +1,106 @@
+"""Append-only job journal (JSONL).
+
+One file per log directory. Each line is an event:
+
+    {"key": "<hash#occ>", "event": "seen",   "no": 7, "command": "..."}
+    {"key": "...",        "event": "oom",    "attempts": 1, "next_vram": 9750}
+    {"key": "...",        "event": "done",   "status": "ok", "returncode": 0,
+     "peak_mib": {"0": 7800}, "avg_util": 84.0}
+
+Folding the events yields per-key state. Terminal statuses are never
+re-dispatched; non-terminal keys remain eligible (this is what makes both
+``--resume``-style restarts and OOM retries fall out of one mechanism).
+The jobs file itself is never written by the scheduler — it stays entirely
+user-owned, and this journal is the scheduler's only persistent state.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+TERMINAL = {"ok", "failed", "timeout", "infeasible", "failed_oom"}
+
+
+@dataclass
+class JobState:
+    no: int | None = None           # persistent display number (first-seen order)
+    command: str = ""
+    attempts: int = 0               # completed attempts so far
+    status: str = "pending"         # pending | running | <terminal>
+    returncode: int | None = None
+    next_vram: int | None = None    # bumped declaration for the next attempt
+    peak_mib: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def terminal(self) -> bool:
+        return self.status in TERMINAL
+
+
+class Journal:
+    def __init__(self, path: str | os.PathLike):
+        self.path = Path(path)
+        self.states: dict[str, JobState] = {}
+        self._next_no = 1
+        if self.path.exists():
+            for line in self.path.read_text().splitlines():
+                if line.strip():
+                    try:
+                        self._fold(json.loads(line))
+                    except (json.JSONDecodeError, KeyError):
+                        continue  # tolerate a torn last line after a crash
+
+    def _fold(self, ev: dict) -> None:
+        st = self.states.setdefault(ev["key"], JobState())
+        kind = ev.get("event")
+        if kind == "seen":
+            st.no = ev.get("no")
+            st.command = ev.get("command", "")
+            self._next_no = max(self._next_no, (st.no or 0) + 1)
+        elif kind == "oom":
+            st.attempts = ev.get("attempts", st.attempts + 1)
+            st.next_vram = ev.get("next_vram", st.next_vram)
+            st.status = "pending"
+        elif kind == "done":
+            st.attempts += 1
+            st.status = ev.get("status", "failed")
+            st.returncode = ev.get("returncode")
+            st.peak_mib = ev.get("peak_mib", {})
+
+    def _append(self, ev: dict) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.path, "a") as f:
+            f.write(json.dumps(ev) + "\n")
+            f.flush()
+            os.fsync(f.fileno())
+
+    # ------------------------------------------------------------- API
+    def state(self, key: str) -> JobState:
+        return self.states.get(key, JobState())
+
+    def ensure_seen(self, key: str, command: str) -> int:
+        """Assign a persistent display number on first sight; return it."""
+        st = self.states.get(key)
+        if st is not None and st.no is not None:
+            return st.no
+        no = self._next_no
+        ev = {"key": key, "event": "seen", "no": no, "command": command}
+        self._append(ev)
+        self._fold(ev)
+        return no
+
+    def record_oom_retry(self, key: str, attempts: int, next_vram: int | None) -> None:
+        ev = {"key": key, "event": "oom", "attempts": attempts, "next_vram": next_vram}
+        self._append(ev)
+        self._fold(ev)
+
+    def record_done(self, key: str, status: str, returncode: int,
+                    peak_mib: dict[int, int], avg_util: float | None) -> None:
+        ev = {
+            "key": key, "event": "done", "status": status, "returncode": returncode,
+            "peak_mib": {str(g): m for g, m in peak_mib.items()}, "avg_util": avg_util,
+        }
+        self._append(ev)
+        self._fold(ev)