ppkit 0.1.0__py3-none-any.whl

pp/commands/queue.py

@@ -0,0 +1,243 @@
+"""Run commands later, under resource limits, in named queues
+
+Each named queue has its own limits, its own run order and its own worker; job ids are machine-wide. A job runs
+when its declared needs fit the resource pools. A pool is slots, GPU memory (reserved per card AND read live
+from nvidia-smi), CPUs, RAM, any counter you name, or anything a command can report as free (a cluster, an API
+quota). A job is retried, timed out and reported as you ask.
+
+    pp queue add -- make docs                          queue a command in the `default` queue, run in this directory
+    pp queue add -q dl --need net=1 -- ./fetch.sh      another queue: its own limits, order and worker
+    pp queue add --timeout 2h --retries 2 --retry-if 'Connection reset|CUDA out of memory' -- python x.py
+    pp queue add --on-fail 'notify-send "job $PP_QUEUE_JOB failed"' -- ./nightly.sh
+    pp queue add --from cmds.txt --prio 5              one job per line (# comments and blank lines skipped)
+    pp queue add --gpu-gb 20 -- python x.py            in a project with a runner (below), run through it
+    pp queue add --no-runner -- make docs              ... or not
+    pp queue add --tag sweep -- python x.py --lr 0.1   a tag names a job, or a group of jobs that share it
+    pp queue wait sweep                                every job tagged sweep (kill, drop, show and log take tags too)
+    pp queue                                           status of every queue: running (last log line), queued, recent
+    pp queue -q dl                                     one queue
+    pp queue show 12            pp queue log 12 -f     every field + log tail / follow the log (across retries;
+                                                       exits like wait)
+    pp queue note 12 "OOM at bs=64; re-added as 15"    leave a note on a job (shown in status and show)
+    pp queue wait 12-15                                block until they end, then print the outcomes (a failure
+                                                       with its log tail); exit 0 ok, 1 any failed or dropped,
+                                                       3 timeout
+    pp queue wait 12 --timeout 30m                     ... giving up after 30 minutes (exit 3)
+    pp queue watch 12                                  a line per event as it happens: start, retry, outcome
+    pp queue wait --idle                               ... until nothing is queued or running
+    pp queue wait --mine                               (an agent) every job this session queued that is live
+                                                       or whose end no wait/watch in this session reported yet
+    pp queue kill 12                                   waits while the worker stops it: killed 12 (stopped on SIGTERM)
+    pp queue drop 13    pp queue retry 11    pp queue reprio 14 0
+    pp queue drop 20-29 -n                             a range or a shared tag: shows what it would do, then asks
+    pp queue hold 15    pp queue release 15    pp queue clean --older 7d
+    pp queue worker status | start -q NAME | stop -q NAME     `add` starts a queue's worker; stop leaves jobs running
+
+Agents: run `pp queue wait ID` in the background (or `pp queue watch ID` under a monitor) and you are told
+how the job ended, without polling and without anything typed into your terminal; `add` prints this hint to
+an agent.
+
+IDs are numbers, ranges (12-15) or a tag (add --tag NAME; without one, the tag is the command, cut to 40
+characters). -q narrows a selection to one queue. A job named by its id must suit the verb, or it is an error; a
+range, or a tag several jobs share, leaves the jobs it does not suit as they are, and kill and drop show such a
+selection and ask first (-y: do not ask, -n: only show). Lower --prio runs first (default 10), ties by id. Each
+queue is strict head-of-line unless its worker runs with backfill, so a big job is never starved by small
+ones. PATH, VIRTUAL_ENV, CONDA_PREFIX and CONDA_DEFAULT_ENV are captured at add time, so `python` means what
+it meant in your shell.
+
+A job's environment, later entries winning: the worker's (that of the `pp queue add` that started it, without an
+agent session's identity); PATH, VIRTUAL_ENV, CONDA_PREFIX and CONDA_DEFAULT_ENV from the shell that added the job;
+the project's [queue] env; add's --env; then PP_QUEUE, PP_QUEUE_JOB, PP_QUEUE_RCFILE and CUDA_VISIBLE_DEVICES. A
+variable you export before `pp queue add` reaches the job only through the worker that add started, so pass a
+token or a secret with --env or the project's env.
+
+Timeouts and retries
+    --timeout DUR      the worker SIGTERMs the job's process group after DUR (90s, 30m, 2h, 1d) and SIGKILLs
+                       it 10s later; the attempt fails with reason "timeout", and what that comes to is
+                       under "How a job ends" below. `none` (or 0): no timeout, even when the queue has a
+                       default one
+    --retries N        re-run a failure up to N more times, with the SAME id: back to the queue after
+    --retry-delay DUR  (default 30s, doubling each attempt, at most 1h). A job waiting out its delay never
+                       holds up the queue. `wait` and `log -f` follow it across attempts.
+    --retry-if REGEX   retry only failures whose reason ("exit 1", "timeout", "lost", ...) or last 200 log lines
+                       match (^ and $ anchor per line); any other failure is final at once (wait, watch
+                       and show say "not retried" and why). `--retry-if '^timeout$'` matches an attempt that
+                       wrote no code of its own. Unset: every failure is retried.
+
+Hooks, on the FINAL outcome of a job
+    --on-fail CMD      a shell command run when the job has failed for good
+    --on-done CMD      ... when it succeeded
+    A hook never fires between retries, and never for a drop. Which hook an outcome owes is under "How a job
+    ends" below.
+    `none` turns a hook off. Hooks run detached in the job's directory with PP_QUEUE PP_QUEUE_JOB
+    PP_QUEUE_EVENT (failed|done) PP_QUEUE_RC PP_QUEUE_REASON PP_QUEUE_LOG PP_QUEUE_CWD PP_QUEUE_TAG; their output
+    goes to <queue>/hooks/<id>-<event>.log.
+
+Per project, from the nearest .pp.toml up from the job's directory:
+
+    [queue]
+    runner = "docker/run.sh"         # put before every command added from this project (relative to the
+                                     # project root); --no-runner skips it
+    env = { HF_HUB_OFFLINE = "1" }   # added to each job's environment; add's --env wins
+
+The job runs in the directory you queued it from (or --cwd), as `RUNNER CMD...`. The runner gets env and
+CUDA_VISIBLE_DEVICES (the card --gpu-gb reserved) in its own environment. A container runner must pass them in
+(docker run -e CUDA_VISIBLE_DEVICES -e HF_HUB_OFFLINE ...), or the job sees every GPU and none of env.
+
+Resources: what a job declares (--gpu-gb, --cpu, --mem-gb, --need NAME=N; every job takes slots = 1) must be
+provided by a pool and fit in EVERY pool that provides it. Shared pools count all queues' running jobs; a
+queue's own pools count only its jobs, so a queue can be held to a share of a shared pool:
+
+    [resources]                  # SHARED by every queue
+    gpu_gb = "auto"              # per card: total - 2 GB, and live free memory from nvidia-smi   (default)
+    cpu = "auto"                 # the machine's CPUs                                              (default)
+    mem_gb = "auto"              # MemTotal                                                        (default)
+    license = 4                  # a counter every queue draws from: --need license=1
+    nodes = { probe = "sinfo -h -t idle -o %D", every = "1m", settle = "2m" }
+                                 # a probe: the amount FREE now (first number the command prints), cached for
+                                 # `every`; jobs launched less than `settle` before a reading are subtracted
+                                 # from it. Add max = N to also cap what pp's jobs may hold.
+    [queue.default.resources]    # this queue's OWN
+    slots = 3                    #                                                                 (default)
+    cpu = 16                     # this queue may use 16 of the shared CPUs
+    gpu_gb = 40                  # ... and hold at most 40 GB per card
+    [queue.dl.resources]
+    net = 2                      # a counter only dl has: --need net=1
+A pool is a number or { max, probe, every, settle }; `false` removes a default. settle is 30s unless set, for
+gpu_gb and every probe: a job launched less than that before a reading may not show in it yet, so its
+reservation is subtracted from the reading.
+
+Config, ~/.config/pp/config.toml: a table per queue, re-read by its worker every loop, with no restart
+needed. An unknown key or a bad value is an error, shown by `pp queue`; while it lasts the worker launches nothing, but
+still records how running jobs end and enforces their timeouts. So does an empty file, which is what a file
+rewritten in place is for a moment; remove it to use the defaults:
+
+    [queue.default]
+    timeout = "6h"       # defaults for jobs added to this queue (add's flags win)
+    retries = 1
+    retry_delay = "1m"
+    retry_if = "Connection reset|503"
+    on_fail = "notify-send 'pp job failed'"     # a shell command; default: none
+    on_done = "none"
+    idle_exit = 600      # the worker exits after this long with nothing queued or running
+    poll = 2             # worker loop period, seconds
+    backfill = false     # let later jobs that fit start while the head waits
+
+A job runs in its own process group, inside an `sh` that writes the job's exit code to a file of that attempt's
+own when the command ends. So the exit code outlives the worker, and a restarted worker adopts running jobs
+instead of losing them. That `sh` is also what says the job is running, for as long as it runs; the command
+itself reads /dev/null, so a process the command leaves behind does not keep the job running. The caller's
+agent session (if any) is recorded as the job's owner and never passed to the job.
+
+A queue's worker is what signals that queue's jobs, and only a worker holds the queue's worker claim. `pp
+queue kill` writes a stop on the job and waits for the outcome that is recorded for it. It makes sure the
+queue has a worker: it wakes the one that is up, or starts one where none is. So a kill with no worker up
+starts one, and that worker carries out the stop and launches the queue's queued jobs. Two kills of one queue
+come to the same thing: the one worker carries out both stops.
+
+`pp queue worker stop` stops the worker that is up. It is not a pause: the next command that needs a worker
+starts one.
+
+A timeout is the worker writing a stop on the job itself, and nothing waits for it, so each step of it goes
+out at the queue's poll. The worker signals the job's process group, escalates to SIGKILL once the grace is
+up, and records how the job ended. Anything left in that group when the run is over goes with it, so a
+command that held the signal back does not outlive its job. One actor ends a job, and a job that ended on its
+own meanwhile keeps its exit code.
+
+How a job ends, whatever reached it. The rules are tried in the order below, and the first that holds for a
+job is what happens to it.
+
+    While a run of the job is live its exit code says nothing about how the job ended: the job's own `sh`
+    holds the run, and the command can write PP_QUEUE_RCFILE itself. What is done about such a job is what
+    its stop asks for.
+
+    A signal pp was refused is recorded as denied, and nothing more goes out for that stop.
+    A stop nobody has signalled yet goes out as SIGTERM to the job's process group.
+    A stop whose grace is up is escalated to SIGKILL, once.
+    A stop inside its grace, or SIGKILLed already, is waited for.
+    A job with no stop whose own timeout has run out gets a timeout's stop.
+    A job with no stop whose time is not up is left as it is.
+
+    Once the run is over the exit code the attempt wrote is the command's last word, and the job ends here. A
+    stop someone asked for is never retried.
+
+    A job a kill's signal stopped is recorded "killed" and fires no hook, unless it wrote its own exit code
+    as the kill arrived, and then it keeps that code and fires the hook for it.
+    A job a kill wrote a stop for that no signal of pp's ever reached ends with its own outcome, the exit
+    code it wrote or the signal it died of, and fires the hook that outcome owes.
+    An exit code of 0 is done, and on_done fires.
+    A timeout's stop pp signalled or was refused, with no exit code of the attempt's own, is a timeout, which
+    may be retried; on a final timeout, on_fail fires.
+    Anything else is the job's own failure, the exit code it wrote or the signal it died of, retried while an
+    attempt is left and --retry-if matches it, and on_fail fires on the last.
+
+One run of a job carries one stop. A kill's ask on a timeout's stop makes it a kill, a second ask keeps what
+has gone out and only shortens the grace, and a refused stop is asked for again. `show` and --json say whose
+stop it is, when it was asked for, what has gone out for it and whether pp was refused, so nobody waits for a
+stop that is not going to be carried out.
+
+A job's record names the files that are its own: its log, the exit code each attempt wrote, and the log of
+each hook that ran for it. `drop` and `clean` take those with the record, so neither leaves a file behind that
+nothing names. State: ~/.local/state/pp/queue/<name>/.
+"""
+from __future__ import annotations
+
+import os
+import shlex
+import shutil
+
+from pp.lib import config, jobq, ui
+
+PROJECT_KEYS = ("runner", "env")
+
+
+def _add_args(p):
+    p.add_argument("--no-runner", action="store_true", help="ignore the project's [queue] runner")
+
+
+def _runner(spec, root: str, path: str | None) -> list[str]:
+    """argv of the project's runner; refuses (exit 2) one that could only fail at launch time, hours later."""
+    if not spec:
+        return []
+    where = f"{root}/.pp.toml"
+    if isinstance(spec, str):
+        argv = shlex.split(spec)
+    elif isinstance(spec, list) and all(isinstance(x, str) for x in spec):
+        argv = list(spec)
+    else:
+        raise config.ConfigError(f"{where}: [queue] runner must be a command line or a list of strings")
+    if not argv:
+        return []
+    exe = os.path.expanduser(argv[0])
+    if not os.path.isabs(exe) and os.sep in exe:
+        exe = os.path.join(root, exe)
+    if os.sep not in exe:
+        if not shutil.which(exe, path=path):
+            ui.die(f"queue: runner {exe} (from {where}) is not on PATH; or pass --no-runner", 2)
+    elif not os.path.exists(exe):
+        ui.die(f"queue: runner {exe} (from {where}) does not exist; or pass --no-runner", 2)
+    elif not os.access(exe, os.X_OK):
+        ui.die(f"queue: runner {exe} (from {where}) is not executable (chmod +x); or pass --no-runner", 2)
+    return [exe] + argv[1:]
+
+
+def _prepare(job: dict, a) -> None:
+    """The project's [queue] runner and env; a job's own --env wins over the project's."""
+    root, data = config.find_project(job["cwd"])
+    if not root:
+        return
+    where = f"{root}/.pp.toml"
+    pcfg = data.get("queue", {})
+    if not isinstance(pcfg, dict):
+        raise config.ConfigError(f"{where}: queue must be a table ([queue] runner = ..., env = {{...}})")
+    config.check_keys(pcfg, f"{where} [queue]", PROJECT_KEYS)
+    env = pcfg.get("env", {})
+    if not isinstance(env, dict):
+        raise config.ConfigError(f"{where}: [queue] env must be a table, e.g. env = {{ HF_HUB_OFFLINE = \"1\" }}")
+    if not getattr(a, "no_runner", False):
+        job["runner"] = _runner(pcfg.get("runner"), root, (job.get("shell_env") or {}).get("PATH"))
+    job["env"] = {**{str(k): str(v) for k, v in env.items()}, **job["env"]}
+
+
+def main(argv: list[str]) -> int:
+    return jobq.run_cli(argv, "queue", "default", __doc__, prepare_job=_prepare, add_args=_add_args)

pp/lib/__init__.py

@@ -0,0 +1 @@
+"""Shared helpers for pp commands. Commands import from here, never from each other."""

pp/lib/config.py

@@ -0,0 +1,110 @@
+"""Configuration: one global TOML file plus an optional per-project `.pp.toml`.
+
+Global   ~/.config/pp/config.toml holds a table per command:  [queue.NAME] ... [http] ... [batch.presets]
+Project  .pp.toml in the project root (nearest one walking up from cwd) holds [promote], [queue], ...
+
+Who owns which table, and what rejects an unknown key in it:
+
+    config.toml  [resources], [queue.NAME]   queue    jobq (a queue's table and [resources])
+                 [http]                      http     http's KEYS
+                 [batch]                     batch    batch._presets
+    .pp.toml     name                        promote and queue (the releases dir, a job's project)
+                 [promote]                   promote  promote._checked
+                 [queue]                     queue    queue._prepare
+                 [NAME] of your own          your command; pp ignores tables it does not own
+
+Both are read-only to pp; edit them by hand. A malformed file is a loud error naming the file, not a silent default.
+"""
+from __future__ import annotations
+
+import os
+import re
+import tomllib
+
+from pp.lib import paths, proc
+
+PROJECT_FILE = ".pp.toml"
+
+
+class ConfigError(Exception):
+    pass
+
+
+def _read(path: str, empty_ok: bool = True) -> dict:
+    """The parsed file; {} when it is absent. With empty_ok False an empty file is an error: rewritten in place
+    (`cat >`, some editors), a file is empty for a moment, and a reader acting on what it says must not take that
+    moment for "no settings"."""
+    try:
+        with open(path, "rb") as f:
+            raw = f.read()
+    except FileNotFoundError:
+        return {}
+    if not raw and not empty_ok:
+        raise ConfigError(f"{path} is empty: being saved, or remove it to use the defaults")
+    try:
+        return tomllib.loads(raw.decode())
+    except (tomllib.TOMLDecodeError, UnicodeDecodeError) as e:
+        raise ConfigError(f"{path}: {e}") from None
+
+
+def global_path() -> str:
+    return os.path.join(paths.config_dir(), "config.toml")
+
+
+def load(section: str | None = None, empty_ok: bool = True) -> dict:
+    """The global config, or one table of it ({} when absent; a ConfigError when it is there but not a table)."""
+    data = _read(global_path(), empty_ok)
+    if not section:
+        return data
+    v = data.get(section, {})
+    if not isinstance(v, dict):
+        raise ConfigError(f"{global_path()}: {section} = {v!r}: [{section}] must be a table")
+    return v
+
+
+def check_keys(table: dict, where: str, known) -> None:
+    """A key pp does not read is a typo until proven otherwise: a ConfigError naming it, never a silent default."""
+    bad = sorted(set(table) - set(known))
+    if bad:
+        raise ConfigError(f"{where}: unknown {', '.join(bad)} (known: {', '.join(known)})")
+
+
+def parse_duration(v, default_unit: str = "s") -> float | None:
+    """90 / 90s / 30m / 2h / 1d (a bare number is default_unit) -> seconds; None for None/''; ValueError otherwise."""
+    if v is None or v == "":
+        return None
+    if isinstance(v, (int, float)) and not isinstance(v, bool):
+        return float(v) * {"s": 1, "m": 60, "h": 3600, "d": 86400}[default_unit]
+    m = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([smhd]?)\s*", str(v))
+    if not m:
+        raise ValueError(f"'{v}' is not a duration like 90s, 30m, 2h, 1d")
+    return float(m[1]) * {"s": 1, "m": 60, "h": 3600, "d": 86400}[m[2] or default_unit]
+
+
+def find_project(start: str | None = None) -> tuple[str | None, dict]:
+    """(root, parsed .pp.toml) of the nearest project file walking up from start; (None, {}) when there is none."""
+    d = os.path.abspath(start or os.getcwd())
+    while True:
+        f = os.path.join(d, PROJECT_FILE)
+        if os.path.isfile(f):
+            return d, _read(f)
+        parent = os.path.dirname(d)
+        if parent == d:
+            return None, {}
+        d = parent
+
+
+def git_root(start: str | None = None) -> str | None:
+    try:
+        out = proc.git("-C", start or os.getcwd(), "rev-parse", "--show-toplevel", group="own", timeout=10)
+    except OSError:
+        return None
+    if out.returncode != 0:
+        return None
+    return out.stdout.strip() or None
+
+
+def project_name(root: str, data: dict | None = None) -> str:
+    """A project's name: `name` at the top of its .pp.toml, else its directory name."""
+    data = data if data is not None else _read(os.path.join(root, PROJECT_FILE))
+    return str(data.get("name") or os.path.basename(os.path.normpath(root)))