jobstate 0.1.0__tar.gz

jobstate-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.pytest_cache/
+.ruff_cache/
+.venv/
+
+# Local planning notes
+docs/design.md
+tasks/PLAN.md

jobstate-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Forschungszentrum Juelich GmbH, Juelich Supercomputing Centre
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

jobstate-0.1.0/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.5
+Name: jobstate
+Version: 0.1.0
+Summary: Submit, watch and report on Slurm jobs from AI agents: an MCP server and CLI for JSC's JUPITER, JURECA, JUWELS and JUSUF
+Project-URL: Repository, https://gitlab.jsc.fz-juelich.de/sdlaml/agents-on-hpc/jobstate-mcp
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai-agents,hpc,jupiter,jureca,jusuf,juwels,mcp,slurm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.12
+Requires-Dist: mcp<3,>=2.3
+Requires-Dist: pyyaml>=6
+Description-Content-Type: text/markdown
+
+# jobstate
+
+Tools for AI agents to submit, watch and report on Slurm jobs on the JSC supercomputers JUPITER, JURECA, JUWELS and JUSUF: an
+[MCP](https://modelcontextprotocol.io) server (`jobstate-mcp`) and a command-line tool (`jobstate`).
+
+Instead of polling `squeue` turn after turn, an agent waits for its jobs with one tool call, or is notified when
+they end; a long job costs one model turn at the end. Submissions go through a dry run and a separate confirmation,
+under limits and budget guards the server enforces itself.
+
+## Install
+
+```sh
+uv tool install jobstate
+```
+
+On a JSC login node, check the setup and add the server to Claude Code:
+
+```sh
+jobstate check
+claude mcp add -s user jobstate -- jobstate-mcp
+```
+
+From your own machine the server reaches the cluster over ssh:
+
+```sh
+jobstate check --cluster jupiter --host <your ssh config name>
+claude mcp add -s user jobstate-jupiter -- jobstate-mcp --cluster jupiter --host <your ssh config name>
+```
+
+## Tools
+
+`job_status`, `list_jobs`, `job_summary`, `tail_log`, `job_report` (GPU efficiency from LLview), `watch_job`,
+`wait_for_jobs`, `cluster_info`, and, unless configured read-only, `submit_job` (a dry run), `confirm_submit` and
+`cancel_job`. End-of-job events carry exit code, run time against the limit, busy cores, peak memory, I/O and
+node-hours from Slurm accounting.
+
+Configuration (allowed accounts, tighter limits, read-only mode, ntfy or webhook notifications) goes into
+`~/.config/jobstate-mcp/config.yaml`. Python 3.12 or later.

jobstate-0.1.0/docs/pypi.md

@@ -0,0 +1,38 @@
+# jobstate
+
+Tools for AI agents to submit, watch and report on Slurm jobs on the JSC supercomputers JUPITER, JURECA, JUWELS and JUSUF: an
+[MCP](https://modelcontextprotocol.io) server (`jobstate-mcp`) and a command-line tool (`jobstate`).
+
+Instead of polling `squeue` turn after turn, an agent waits for its jobs with one tool call, or is notified when
+they end; a long job costs one model turn at the end. Submissions go through a dry run and a separate confirmation,
+under limits and budget guards the server enforces itself.
+
+## Install
+
+```sh
+uv tool install jobstate
+```
+
+On a JSC login node, check the setup and add the server to Claude Code:
+
+```sh
+jobstate check
+claude mcp add -s user jobstate -- jobstate-mcp
+```
+
+From your own machine the server reaches the cluster over ssh:
+
+```sh
+jobstate check --cluster jupiter --host <your ssh config name>
+claude mcp add -s user jobstate-jupiter -- jobstate-mcp --cluster jupiter --host <your ssh config name>
+```
+
+## Tools
+
+`job_status`, `list_jobs`, `job_summary`, `tail_log`, `job_report` (GPU efficiency from LLview), `watch_job`,
+`wait_for_jobs`, `cluster_info`, and, unless configured read-only, `submit_job` (a dry run), `confirm_submit` and
+`cancel_job`. End-of-job events carry exit code, run time against the limit, busy cores, peak memory, I/O and
+node-hours from Slurm accounting.
+
+Configuration (allowed accounts, tighter limits, read-only mode, ntfy or webhook notifications) goes into
+`~/.config/jobstate-mcp/config.yaml`. Python 3.12 or later.

jobstate-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "jobstate"
+version = "0.1.0"
+description = "Submit, watch and report on Slurm jobs from AI agents: an MCP server and CLI for JSC's JUPITER, JURECA, JUWELS and JUSUF"
+readme = "docs/pypi.md"  # the PyPI page; README.md is the repository's proposal and guide
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+dependencies = ["mcp>=2.3,<3", "pyyaml>=6"]
+keywords = ["mcp", "slurm", "hpc", "ai-agents", "jupiter", "jureca", "juwels", "jusuf"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "Operating System :: POSIX",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: System :: Distributed Computing",
+]
+
+[project.urls]
+Repository = "https://gitlab.jsc.fz-juelich.de/sdlaml/agents-on-hpc/jobstate-mcp"
+
+[project.scripts]
+jobstate = "jobstate.cli:main"
+jobstate-mcp = "jobstate.mcp.server:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jobstate"]
+
+[tool.hatch.build.targets.sdist]
+# Public archive: code, licence and the PyPI description only. tests/fixtures hold recorded cluster output (user
+# and project names, paths), README.md, tasks/ and docs/ internal notes; none of it belongs on PyPI.
+include = ["src/jobstate", "docs/pypi.md", "LICENSE", "pyproject.toml"]
+exclude = [".zenv"]  # the local environment; its packages' LICENSE files would match license-files
+
+[dependency-groups]
+dev = [
+    "pytest>=9",
+]

jobstate-0.1.0/src/jobstate/__init__.py

@@ -0,0 +1 @@
+"""jobstate: submit, watch and report on HPC jobs. Core library shared by the CLI, MCP server and Python API."""

jobstate-0.1.0/src/jobstate/api.py

@@ -0,0 +1 @@
+"""Python API front end. Calls core functions only; never another front end."""

jobstate-0.1.0/src/jobstate/cli.py

@@ -0,0 +1,158 @@
+"""Command-line front end (`jobstate`). Calls core functions only; never another front end.
+
+    jobstate status 123456 123457_2      one line per job
+    jobstate list [--state RUNNING]      your queued and running jobs
+    jobstate tail 123456 [--stderr]      the end of a job's log
+    jobstate report 123456               efficiency from LLview, with the data's age
+    jobstate info                        partitions, limits, filesystems and rules of the cluster
+    jobstate guards [--reset SCRIPT]     what agents may submit, budget guard use; allow a blocked script again
+    jobstate check                       can jobstate work here: ssh, Slurm, sacct, LLview, config, state
+
+`--cluster` defaults to $SYSTEMNAME on JSC machines. `--exec auto` runs Slurm locally where `squeue` exists and
+over ssh otherwise; `--host` is the ssh destination (a ~/.ssh/config name works), default the login alias. Over
+ssh, commands go to the cluster's login nodes in turn, skipping unreachable ones (`--no-failover`: only --host).
+Exit status: 0 done, 1 the cluster or a job could not answer, 2 bad arguments.
+"""
+
+import argparse
+import asyncio
+import json
+import sys
+from collections.abc import Sequence
+
+from .core import capabilities
+from .core import config as configs
+from .core import profile as profiles
+from .core import slurm
+from .core.llview import Report
+from .core.policy import Ledger, Policy
+from .core.service import EXEC_MODES, JobNotFound, Service, Tail, default_cluster, explain, plain
+
+
+class _Usage(Exception):
+    """Bad arguments found after parsing (exit status 2)."""
+
+
+def _parser() -> argparse.ArgumentParser:
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--cluster", choices=profiles.available(), default=default_cluster(),
+                        help="cluster profile (default: $SYSTEMNAME)")
+    common.add_argument("--exec", dest="exec_mode", choices=EXEC_MODES, default="auto",
+                        help="run Slurm commands here or over ssh (default: auto)")
+    common.add_argument("--host", help="ssh destination, e.g. a ~/.ssh/config name (default: the login alias)")
+    common.add_argument("--no-failover", dest="failover", action="store_false",
+                        help="use only --host, not the cluster's other login nodes")
+    common.add_argument("--json", action="store_true", help="print JSON instead of text")
+
+    parser = argparse.ArgumentParser(prog="jobstate", description="Status, queue and logs of Slurm jobs.")
+    sub = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
+    p = sub.add_parser("status", parents=[common], help="state of jobs, one line each")
+    p.add_argument("ids", nargs="+", metavar="JOBID")
+    p = sub.add_parser("list", parents=[common], help="your queued and running jobs")
+    p.add_argument("--state", action="append", type=str.upper, choices=sorted(slurm.STATES), metavar="STATE",
+                   help="only jobs in this state or with this flag (repeatable)")
+    p = sub.add_parser("tail", parents=[common], help="the end of a job's log")
+    p.add_argument("id", metavar="JOBID")
+    p.add_argument("--stderr", action="store_const", const="stderr", default="stdout", dest="stream",
+                   help="the stderr log (the stdout log if the job writes both into one)")
+    p.add_argument("-n", "--lines", type=int, default=50, help=f"lines, at most {slurm.MAX_LOG_LINES} (default 50)")
+    p.add_argument("--bytes", type=int, default=16384, dest="max_bytes",
+                   help=f"read at most this much, up to {slurm.MAX_LOG_BYTES} (default 16384)")
+    p.add_argument("--offset", type=int, help="continue from this byte offset (printed by an earlier call)")
+    p = sub.add_parser("report", parents=[common], help="a job's efficiency from LLview")
+    p.add_argument("id", metavar="JOBID")
+    sub.add_parser("info", parents=[common], help="partitions, limits, filesystems and rules of the cluster")
+    sub.add_parser("check", parents=[common], help="check ssh, Slurm, sacct, LLview, config and state, with fixes")
+    p = sub.add_parser("guards", parents=[common], help="what agents may submit, and the budget guards' use")
+    p.add_argument("--reset", metavar="SCRIPT", help="allow a script blocked after failing too often (the script's "
+                                                     "path on the cluster, as shown)")
+    return parser
+
+
+def _tail_note(t: Tail) -> str:
+    c = t.chunk
+    notes = [f"{t.job.id} {t.stream} {c.path}", f"bytes {c.start}-{c.next_offset} of {c.size}"]
+    if t.merged:
+        notes.append("stderr goes to this log")
+    if c.reset:
+        notes.append("the log was shorter than the offset; read from its end")
+    if c.truncated:
+        notes.append("more after this" if c.next_offset < c.size else "earlier output left out")
+    return f"== {'; '.join(notes)}; next: --offset {c.next_offset}"
+
+
+async def _run(args, service: Service) -> int:
+    out = sys.stdout
+    if args.command == "check":
+        checks = await capabilities.check(service)
+        out.write(f"{service.profile.name}\n" + "".join(f"  {c.line()}\n" for c in checks))
+        return 0 if all(c.ok is not False for c in checks) else 1
+    if args.command == "guards":
+        policy = Policy(service.profile, configs.load(service.profile), Ledger(service.profile.cluster))
+        if args.reset:
+            policy.ledger.reset(args.reset)
+            out.write(f"{args.reset}: failures before now no longer count\n")
+            return 0
+        out.write(f"{service.profile.name} ({configs.default_path()}"
+                  f"{'' if configs.default_path().exists() else ', not found: defaults'})\n")
+        out.writelines(f"  {line}\n" for line in policy.describe())
+        return 0
+    if args.command == "info":
+        out.write(json.dumps(plain(service.profile), indent=2) + "\n" if args.json
+                  else profiles.context(service.profile))
+        return 0
+
+    unknown: list[str] = []
+    if args.command == "status":
+        result = await service.status(args.ids)
+        unknown = slurm.missing(args.ids, result.value or [])
+    elif args.command == "list":
+        result = await service.jobs(args.state)
+    elif args.command == "report":
+        result = await service.report(args.id)
+    else:
+        result = await service.tail(args.id, stream=args.stream, lines=args.lines, max_bytes=args.max_bytes,
+                                    offset=args.offset)
+    if result.value is None:
+        print(f"jobstate: {explain(result.outcome)}", file=sys.stderr)
+        return 1
+
+    value = result.value
+    if args.json:
+        out.write(json.dumps(plain(value), indent=2) + "\n")
+    elif isinstance(value, Report):
+        out.write(value.line() + "\n")
+    elif isinstance(value, Tail):
+        print(_tail_note(value), file=sys.stderr)  # the log itself is job output: keep it apart from our notes
+        out.write(value.chunk.text)
+    else:
+        out.writelines(f"{job.line()}\n" for job in value)
+        if not value and args.command == "list":
+            print("no jobs in the queue", file=sys.stderr)
+
+    out.flush()  # our messages on stderr come after the lines they follow
+    for job_id in unknown:
+        print(f"jobstate: no job {job_id}", file=sys.stderr)
+    return 1 if unknown else 0
+
+
+def main(argv: Sequence[str] | None = None, *, service: Service | None = None) -> int:
+    """Entry point. Tests pass a `service` built on a fake runner instead of a real cluster connection."""
+    args = _parser().parse_args(argv)
+    try:
+        if service is None:
+            if args.cluster is None:
+                raise _Usage(f"--cluster is required here (one of: {', '.join(profiles.available())})")
+            service = Service.create(args.cluster, exec_mode=args.exec_mode, host=args.host,
+                                     failover=args.failover)
+        return asyncio.run(_run(args, service))
+    except (_Usage, ValueError) as e:
+        print(f"jobstate: {e}", file=sys.stderr)
+        return 2
+    except JobNotFound as e:
+        print(f"jobstate: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

jobstate-0.1.0/src/jobstate/core/__init__.py

@@ -0,0 +1 @@
+"""Core library: everything the front ends share. No MCP, CLI or I/O-format concerns here."""

jobstate-0.1.0/src/jobstate/core/capabilities.py

@@ -0,0 +1,86 @@
+"""Capability check per cluster: ssh, Slurm --json, sacct, the LLview CLI, the user config and the state directory.
+
+`jobstate check` runs it and explains each problem; `jobstate-mcp` runs it in the background at startup and shows the
+result in `cluster_info`. A failed check never stops the server (chosen 2026-10-17): a cluster in maintenance or a
+broken LLview must not take the tools away; the tools then return their own clear errors.
+"""
+
+from dataclasses import dataclass
+
+from . import config as configs
+from .outcome import Outcome, Status
+from .service import Service, ssh_hint
+from .slurm import ParseError, parse_sacct, parse_squeue, sacct_recent, squeue_mine
+from .state import Store
+
+TIMEOUT = 30.0
+
+@dataclass(frozen=True)
+class Check:
+    name: str
+    ok: bool | None  # None: could not be checked (e.g. the cluster was unreachable)
+    detail: str
+    fix: str | None = None
+
+    def line(self) -> str:
+        mark = {True: "ok  ", False: "FAIL", None: "?   "}[self.ok]
+        return f"{mark} {self.name}: {self.detail}" + (f"\n     fix: {self.fix}" if self.fix else "")
+
+
+async def check(service: Service, *, store: Store | None = None) -> list[Check]:
+    """Every check for `service`'s cluster, in order; later cluster checks are skipped if ssh fails."""
+    p, results = service.profile, []
+
+    try:
+        read_only = configs.load(p).read_only
+        results.append(Check("config", True, f"{configs.default_path()} " + (
+            "loaded" if configs.default_path().exists() else "not found: defaults (writes within site limits)")
+            + (" (read-only)" if read_only else "")))
+    except configs.ConfigError as e:
+        results.append(Check("config", False, str(e), "fix the key named above; `jobstate guards` shows the result"))
+
+    try:
+        state = store or Store(p.cluster)
+        state.heartbeat()
+        state.end()
+        results.append(Check("state", True, f"{state.dir} is writable"))
+    except OSError as e:
+        results.append(Check("state", False, str(e), "jobstate keeps its state under $XDG_STATE_HOME or "
+                                                     "~/.local/state; make it writable"))
+
+    first = await service.runner.run(["true"], timeout=TIMEOUT)
+    where = service.runner.host or "this machine"
+    if not first.ok:
+        hint = ssh_hint(first)
+        results.append(Check("connection", False, f"{where}: {first.message or first.stderr.strip()[:200]}",
+                             hint or "run `ssh <host> true` in a terminal to see what ssh needs"))
+        return results + [Check(name, None, "skipped: no connection") for name in ("slurm", "sacct", "llview")]
+    results.append(Check("connection", True, f"commands run on {where} ({first.elapsed:.1f} s)"))
+
+    squeue = await service.runner.run(squeue_mine(), timeout=TIMEOUT)
+    results.append(_parsed("slurm", squeue, parse_squeue, "squeue --me --json",
+                           "Slurm 21.08 or later is needed for --json; is this a Slurm login node?"))
+    sacct = await service.runner.run(sacct_recent(), timeout=TIMEOUT)
+    results.append(_parsed("sacct", sacct, parse_sacct, "sacct --json",
+                           "job status after a job leaves the queue needs Slurm accounting (slurmdbd)"))
+
+    if p.llview is None:
+        results.append(Check("llview", None, "no LLview CLI in the profile: job_report unavailable"))
+    else:
+        cli = await service.runner.run(["test", "-x", p.llview.cli], timeout=TIMEOUT)
+        if cli.ok:
+            results.append(Check("llview", True, f"{p.llview.cli} found (job_report; data 10-35 min behind)"))
+        else:
+            results.append(Check("llview", False, f"{p.llview.cli} not found", "job_report will not work; the other "
+                                 "tools do not need LLview"))
+    return results
+
+
+def _parsed(name: str, out: Outcome, parse, command: str, fix: str) -> Check:
+    if not out.ok:
+        return Check(name, False, f"{command}: {out.message or (out.stderr.strip().splitlines() or ['?'])[-1]}", fix)
+    try:
+        jobs = parse(out.stdout)
+    except ParseError as e:
+        return Check(name, False, f"{command}: {e}", fix)
+    return Check(name, True, f"{command} works ({len(jobs)} job{'s' if len(jobs) != 1 else ''}, {out.elapsed:.1f} s)")

jobstate-0.1.0/src/jobstate/core/config.py

@@ -0,0 +1,142 @@
+"""The user's config (~/.config/jobstate-mcp/config.yaml): per cluster, what agents may do within the site limits.
+
+    jupiter:
+      read_only: false
+      accounts: [solai]            # budgets agents may use; missing: any account Slurm accepts
+      default_account: solai       # used when a request names none
+      partitions: [booster]        # missing: every partition in the profile
+      max_nodes: 4                 # per job; missing: the partition's limit
+      max_time: "02:00:00"         # per job; missing: the partition's limit
+      paths:                       # where scripts and working directories may be; missing: the account's
+        - $SCRATCH_solai           #   project and scratch directories from the cluster profile
+        - /e/scratch/solai/me      #   ($PROJECT_<account>, ...; variables are expanded on the cluster)
+      guards:
+        submissions_per_hour: 60
+        concurrent_jobs: 50
+        node_hours_per_day: 500
+        failures_in_a_row: 5
+      notify:                      # copies of every end-of-job event, beside the agent's own delivery
+        ntfy: https://ntfy.sh/<a secret topic>   # phone notification with the event's text
+        webhook: https://example.org/hook        # the event as JSON
+        command: [notify-send, jobstate]         # the event as JSON on stdin
+
+The config can only tighten the cluster profile: a limit above the site's is an error, not ignored. Without a
+config, agents may submit within the site limits with any account Slurm accepts, under the default guards.
+"""
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+from .delivery import NotifyConfig
+from .profile import Profile
+from .schema import SchemaError, Section, format_time
+
+# Default guards (chosen 2026-10-17): loose, for supervised sweeps; the config can tighten them.
+SUBMISSIONS_PER_HOUR = 60
+CONCURRENT_JOBS = 50
+NODE_HOURS_PER_DAY = 500
+FAILURES_IN_A_ROW = 5
+
+
+class ConfigError(SchemaError):
+    """The config file is malformed, breaks the schema or loosens a site limit."""
+
+
+class _Section(Section):
+    error = ConfigError
+
+
+@dataclass(frozen=True)
+class Guards:
+    submissions_per_hour: int = SUBMISSIONS_PER_HOUR
+    concurrent_jobs: int = CONCURRENT_JOBS  # jobs this server submitted, queued or running
+    node_hours_per_day: int = NODE_HOURS_PER_DAY  # requested nodes x walltime, last 24 h
+    failures_in_a_row: int = FAILURES_IN_A_ROW  # of one script; then refused until a person resets it
+
+
+@dataclass(frozen=True)
+class ClusterConfig:
+    read_only: bool = False
+    accounts: tuple[str, ...] = ()  # empty: any account Slurm accepts
+    default_account: str | None = None
+    partitions: tuple[str, ...] = ()  # empty: every partition in the profile
+    max_nodes: int | None = None
+    max_time: int | None = None  # minutes
+    paths: tuple[str, ...] = ()  # directories, may hold $VARs (expanded on the cluster); empty: from the profile
+    guards: Guards = field(default_factory=Guards)
+    notify: NotifyConfig = field(default_factory=NotifyConfig)
+
+
+def default_path() -> Path:
+    config_home = os.environ.get("XDG_CONFIG_HOME")
+    return (Path(config_home) if config_home else Path.home() / ".config") / "jobstate-mcp" / "config.yaml"
+
+
+def _cluster(s: _Section, profile: Profile) -> ClusterConfig:
+    accounts = tuple(s.strings("accounts")) if "accounts" in s.data else ()
+    partitions = tuple(s.strings("partitions")) if "partitions" in s.data else ()
+    paths = tuple(s.strings("paths")) if "paths" in s.data else ()
+    g = s.opt_section("guards")
+    guards = Guards()
+    if g is not None:
+        guards = Guards(*(g.opt_count(name, default=getattr(guards, name), minimum=1)
+                          for name in ("submissions_per_hour", "concurrent_jobs", "node_hours_per_day",
+                                       "failures_in_a_row")))
+        g.done()
+    n = s.opt_section("notify")
+    notify = NotifyConfig()
+    if n is not None:
+        notify = NotifyConfig(webhook=n.opt_text("webhook"), ntfy=n.opt_text("ntfy"),
+                              command=tuple(n.strings("command")) if "command" in n.data else ())
+        n.done()
+        for key, url in (("webhook", notify.webhook), ("ntfy", notify.ntfy)):
+            if url and not url.startswith(("https://", "http://localhost", "http://127.0.0.1")):
+                raise ConfigError(f"{n.path}.{key}: {url!r} must be https:// (or http:// on localhost)")
+    config = ClusterConfig(read_only=s.flag("read_only") if "read_only" in s.data else False, accounts=accounts,
+                           default_account=s.opt_text("default_account"), partitions=partitions,
+                           max_nodes=s.opt_count("max_nodes", default=None, minimum=1),
+                           max_time=s.time("max_time") if "max_time" in s.data else None, paths=paths, guards=guards,
+                           notify=notify)
+    s.done()
+
+    for name in partitions:
+        if name not in profile.partitions:
+            raise ConfigError(f"{s.path}.partitions: {name!r} is not a partition of {profile.name} "
+                              f"({', '.join(profile.partitions)})")
+    if config.default_account and accounts and config.default_account not in accounts:
+        raise ConfigError(f"{s.path}.default_account: {config.default_account!r} is not in `accounts`")
+    usable = [profile.partitions[p] for p in partitions or profile.partitions]
+    if config.max_nodes is not None and config.max_nodes > max(p.max_nodes for p in usable):
+        raise ConfigError(f"{s.path}.max_nodes: {config.max_nodes} is above the site limit "
+                          f"({max(p.max_nodes for p in usable)}); the config can only tighten limits")
+    if config.max_time is not None and config.max_time > max(p.max_time for p in usable):
+        raise ConfigError(f"{s.path}.max_time: {format_time(config.max_time)} is above the site limit "
+                          f"({format_time(max(p.max_time for p in usable))}); the config can only tighten limits")
+    for path in paths:
+        if not (path.startswith("/") or path.startswith("$")):
+            raise ConfigError(f"{s.path}.paths: {path!r} must be absolute or start with a $VARIABLE")
+    return config
+
+
+def load(profile: Profile, path: Path | None = None) -> ClusterConfig:
+    """The config for `profile`'s cluster; the defaults when the file or the cluster's entry is missing."""
+    path = path or default_path()
+    try:
+        data = yaml.safe_load(path.read_text())
+    except FileNotFoundError:
+        return ClusterConfig()
+    except yaml.YAMLError as e:
+        raise ConfigError(f"{path}: not valid YAML: {e}") from None
+    if data is None:
+        return ClusterConfig()
+    if not isinstance(data, dict):
+        raise ConfigError(f"{path}: expected a mapping of cluster names")
+    if profile.cluster not in data:
+        return ClusterConfig()
+    try:
+        return _cluster(_Section(data[profile.cluster], profile.cluster), profile)
+    except ConfigError as e:
+        raise ConfigError(f"{path}: {e}") from None

jobstate-0.1.0/src/jobstate/core/delivery.py

@@ -0,0 +1,86 @@
+"""Event delivery to observers beside the agent: a webhook (JSON POST), ntfy (a phone notification) or a command.
+
+Configured per cluster in the user's jobstate config (`notify:`); nothing is sent anywhere otherwise. Observers get
+a copy: they never mark an event delivered, so the agent still receives it (wait_for_jobs, channel events). They
+carry the event's structured fields and text only, never job names or log text. A failed delivery is logged and
+dropped, never retried in a loop and never allowed to stop the watcher.
+"""
+
+import asyncio
+import json
+import logging
+import urllib.request
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from .events import Event, event_dict
+from .profile import Profile
+
+log = logging.getLogger(__name__)
+
+TIMEOUT = 10.0  # seconds per delivery
+
+
+@dataclass(frozen=True)
+class NotifyConfig:
+    webhook: str | None = None  # URL; receives the event as JSON
+    ntfy: str | None = None  # ntfy topic URL, e.g. https://ntfy.sh/<secret topic>; receives the text
+    command: tuple[str, ...] = ()  # argv; receives the event as JSON on stdin
+
+
+def _post(url: str, body: bytes, headers: dict[str, str]) -> int:
+    request = urllib.request.Request(url, data=body, headers=headers, method="POST")
+    with urllib.request.urlopen(request, timeout=TIMEOUT) as response:  # noqa: S310 (URL from the user's config)
+        return response.status
+
+
+class Webhook:
+    def __init__(self, url: str, profile: Profile):
+        self.url, self.profile = url, profile
+
+    async def __call__(self, event: Event) -> None:
+        body = json.dumps(event_dict(event, self.profile)).encode()
+        await asyncio.to_thread(_post, self.url, body, {"Content-Type": "application/json"})
+
+
+class Ntfy:
+    def __init__(self, url: str, profile: Profile):
+        self.url, self.profile = url, profile
+
+    async def __call__(self, event: Event) -> None:
+        failed = sum(1 for j in event.jobs if not j.ok)
+        title = f"jobstate {event.cluster}: {len(event.jobs)} job{'s' if len(event.jobs) > 1 else ''} done" + (
+            f", {failed} not COMPLETED" if failed else "")
+        headers = {"Title": title, "Tags": "warning" if failed else "white_check_mark"}
+        await asyncio.to_thread(_post, self.url, event.text(self.profile).encode(), headers)
+
+
+class Command:
+    def __init__(self, argv: Sequence[str], profile: Profile):
+        self.argv, self.profile = tuple(argv), profile
+
+    async def __call__(self, event: Event) -> None:
+        proc = await asyncio.create_subprocess_exec(*self.argv, stdin=asyncio.subprocess.PIPE,
+                                                    stdout=asyncio.subprocess.DEVNULL,
+                                                    stderr=asyncio.subprocess.DEVNULL)
+        try:
+            async with asyncio.timeout(TIMEOUT):
+                await proc.communicate(json.dumps(event_dict(event, self.profile)).encode())
+        except TimeoutError:
+            proc.kill()
+            await proc.wait()
+            raise
+        if proc.returncode:
+            raise RuntimeError(f"{self.argv[0]} exited with {proc.returncode}")
+
+
+def sinks(config: NotifyConfig, profile: Profile) -> list:
+    """The observers configured for a cluster, each an async callable taking an Event."""
+    found: list = []
+    if config.webhook:
+        found.append(Webhook(config.webhook, profile))
+    if config.ntfy:
+        found.append(Ntfy(config.ntfy, profile))
+    if config.command:
+        found.append(Command(config.command, profile))
+    return found