nomadctl 0.2.0__py3-none-any.whl

nd/commands/list.py

@@ -0,0 +1,120 @@
+"""The ``nd list`` command: list known job files against live cluster state."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+from nclutils import pp
+
+from nd.commands._common import VerboseOption, configure_verbosity
+from nd.jobfiles import discover_job_files, load_job_directories
+from nd.nomad import NomadClient, NomadConfig
+from nd.ui.links import WebUi
+from nd.ui.panels import status_table, titled_panel
+from nd.ui.styles import status_cell
+
+if TYPE_CHECKING:
+    from nd.jobfiles import JobFile
+    from nd.nomad.models.job import JobListStub
+
+# Cluster-status label for a job file whose name is not present in Nomad at all.
+_NOT_DEPLOYED = "not deployed"
+
+
+@dataclass(frozen=True)
+class ListRow:
+    """One rendered row: a job file's name, path, and cluster status."""
+
+    job_name: str
+    path: str
+    cluster_status: str
+    # Nomad job ID for the web UI link, or None when the job is not deployed.
+    link_id: str | None
+
+
+def build_rows(
+    files: list[JobFile], jobs: list[JobListStub], *, hide_running: bool = False
+) -> list[ListRow]:
+    """Join discovered job files to cluster jobs by name, classifying each.
+
+    A file with no resolved job name still appears (named ``?``) so unresolved
+    interpolated names are visible rather than silently dropped. A deployed job
+    carries its Nomad ID so its name can be linked to the web UI.
+
+    Args:
+        files: Discovered job files to classify.
+        jobs: Live cluster jobs to join against.
+        hide_running: When True, omit jobs whose cluster status is ``running`` so
+            only dead and not-deployed files remain.
+
+    Returns:
+        Sorted list of rows, one per job name per file.
+    """
+    jobs_by_name = {job.name: job for job in jobs}
+    rows: list[ListRow] = []
+    for jf in files:
+        names = jf.job_names or ["?"]
+        for name in names:
+            job = jobs_by_name.get(name)
+            status = job.status if job else _NOT_DEPLOYED
+            if hide_running and status == "running":
+                continue
+            link_id = job.id if job else None
+            rows.append(
+                ListRow(job_name=name, path=str(jf.path), cluster_status=status, link_id=link_id)
+            )
+    return sorted(rows, key=lambda r: r.job_name)
+
+
+def _render(rows: list[ListRow], ui_base: str) -> None:
+    """Print the job-file table inside a titled panel, linking deployed jobs to the web UI."""
+    if not rows:
+        pp.info("No job files found; set [jobs] directories in your nd config.")
+        return
+    web = WebUi(ui_base)
+    table = status_table("JOB", "STATUS", "FILE")
+    for row in rows:
+        name = web.job(row.link_id, row.job_name) if row.link_id else row.job_name
+        # "not deployed" is not a Nomad status, so style it muted rather than via status_cell.
+        cell = (
+            status_cell(row.cluster_status)
+            if row.cluster_status != _NOT_DEPLOYED
+            else "[dim]• not deployed[/]"
+        )
+        table.add_row(name, cell, row.path)
+    pp.console().print(titled_panel(table, "Job files"))
+
+
+app = typer.Typer()
+
+
+@app.callback(invoke_without_command=True)
+def list_(
+    ctx: typer.Context,
+    hide_running: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option(
+            "--hide-running",
+            "-R",
+            help="Hide jobs that are currently running, leaving only dead and not-deployed files.",
+        ),
+    ] = False,
+    verbose: VerboseOption = 0,
+) -> None:
+    """List known job files and whether each is running, dead, or not deployed."""
+    configure_verbosity(ctx, verbose)
+    asyncio.run(_run(hide_running=hide_running))
+
+
+async def _run(*, hide_running: bool = False) -> None:
+    """Discover job files, fetch cluster jobs, and render the joined table."""
+    directories = load_job_directories()
+    files = discover_job_files(directories)
+    pp.debug(f"Discovered {len(files)} job file(s) in {len(directories)} dir(s)")
+    config = NomadConfig.resolve()
+    async with NomadClient.from_config(config) as client:
+        jobs = await client.jobs.list()
+    _render(build_rows(files, jobs, hide_running=hide_running), config.ui_base)

nd/commands/logs.py

@@ -0,0 +1,76 @@
+"""The ``nd logs`` command: stream, tail, or export a running task's logs."""
+
+from __future__ import annotations
+
+from pathlib import Path  # noqa: TC003
+from typing import Annotated
+
+import typer
+
+from nd.commands._common import VerboseOption, configure_verbosity, run_alloc_action
+from nd.nomad import NomadConfig
+
+# allow_interspersed_args lets options follow the positional JOB (e.g. `nd logs web -e`).
+app = typer.Typer(context_settings={"allow_interspersed_args": True})
+
+
+def _streams(*, only_stdout: bool, only_stderr: bool) -> tuple[str, ...]:
+    """Resolve the stream-selection flags to the streams to read (default both)."""
+    if only_stdout and not only_stderr:
+        return ("stdout",)
+    if only_stderr and not only_stdout:
+        return ("stderr",)
+    return ("stdout", "stderr")
+
+
+@app.callback(invoke_without_command=True)
+def logs(  # noqa: PLR0913
+    ctx: typer.Context,
+    job: Annotated[
+        str | None,
+        typer.Argument(
+            help="Running job to read; matches any job whose name starts with this. "
+            "Omit to pick from a list."
+        ),
+    ] = None,
+    task: Annotated[
+        str | None,
+        typer.Option("--task", "-t", help="Target task; skips the task prompt."),
+    ] = None,
+    only_stdout: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option("--stdout", "-o", help="Show only the stdout stream."),
+    ] = False,
+    only_stderr: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option("--stderr", "-e", help="Show only the stderr stream."),
+    ] = False,
+    tail: Annotated[
+        int | None,
+        typer.Option("--tail", "-n", help="Show the last N lines, static (no follow)."),
+    ] = None,
+    export: Annotated[
+        Path | None,
+        typer.Option("--export", help="Write current logs to this file, then exit."),
+    ] = None,
+    verbose: VerboseOption = 0,
+) -> None:
+    """Stream a task's logs, or tail/export them.
+
+    Defaults to a live stream of both stdout and stderr (interleaved) until
+    interrupted with Ctrl-C. Pass --stdout or --stderr to show a single stream.
+    """
+    configure_verbosity(ctx, verbose)
+    config = NomadConfig.resolve()
+    streams = _streams(only_stdout=only_stdout, only_stderr=only_stderr)
+    # running_only=False so logs of a dead, completed, or failed task stay reachable
+    # (debugging a crash is the main reason to read logs).
+    run_alloc_action(
+        config,
+        job=job,
+        task=task,
+        running_only=False,
+        action=lambda nomad, alloc_id, task_name: nomad.stream_logs(
+            alloc_id, task_name, streams=streams, tail=tail, export_path=export
+        ),
+    )

nd/commands/plan.py

@@ -0,0 +1,103 @@
+"""The ``nd plan`` command: preview job-file changes via `nomad job plan`."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+from nclutils import pp
+
+from nd.binary import NomadBinary, NomadBinaryError
+from nd.commands._common import VerboseOption, configure_verbosity
+from nd.jobfiles import candidates_for, discover_job_files, load_job_directories
+from nd.nomad import NomadConfig
+from nd.targets import resolve_targets, select_candidates
+
+if TYPE_CHECKING:
+    from nd.jobfiles import JobCandidate
+    from nd.targets import TargetResolution
+
+
+# allow_interspersed_args lets options follow the positional JOB (e.g. `nd plan web -n`);
+# Typer groups disable that by default, which would parse `-n` as a subcommand.
+app = typer.Typer(context_settings={"allow_interspersed_args": True})
+
+
+@app.callback(invoke_without_command=True)
+def plan(
+    ctx: typer.Context,
+    job: Annotated[
+        str | None,
+        typer.Argument(
+            help="Job to plan; matches any job whose name starts with this. "
+            "Omit to pick from a list."
+        ),
+    ] = None,
+    dry_run: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option("--dry-run", "-n", help="Resolve and report targets without planning them."),
+    ] = False,
+    verbose: VerboseOption = 0,
+) -> None:
+    """Preview the changes one or more job files would apply, including to running jobs."""
+    configure_verbosity(ctx, verbose)
+    exit_code = asyncio.run(_run(job_arg=job, dry_run=dry_run))
+    if exit_code != 0:
+        raise typer.Exit(exit_code)
+
+
+async def _run(*, job_arg: str | None, dry_run: bool) -> int:
+    """Resolve candidates (all files), then validate + plan each selected one."""
+    files = discover_job_files(load_job_directories())
+    candidates = candidates_for(files)
+    if not candidates:
+        pp.info("No job files found; set [jobs] directories in your nd config.")
+        return 0
+
+    resolution: TargetResolution[JobCandidate] = resolve_targets(
+        candidates, job_arg, name_of=lambda c: c.name
+    )
+    targets = await select_candidates(
+        resolution, "Select jobs to plan", label_of=lambda c: f"{c.name}  [{c.file.path.name}]"
+    )
+    if targets is None:
+        return 0
+    if not targets:
+        pp.error(f"No job file matching '{job_arg}'")
+        return 1
+
+    if dry_run:
+        for c in targets:
+            pp.dryrun(f"would plan {c.name} ({c.file.path})")
+        return 0
+
+    return _plan_all(targets)
+
+
+def _plan_all(targets: list[JobCandidate]) -> int:
+    """Validate then plan each unique file, surfacing `nomad job plan` verbatim.
+
+    Returns 0 when every plan ran (including "changes present"); 1 if any file
+    failed validation or the binary could not run.
+    """
+    # Resolve config so the binary targets the same cluster as nd (including
+    # config-file overrides), not just whatever NOMAD_* env vars are ambient.
+    config = NomadConfig.resolve()
+    try:
+        nomad = NomadBinary.create(config)
+    except NomadBinaryError as exc:
+        pp.error(str(exc))
+        return 1
+
+    failures = 0
+    # dict.fromkeys dedups while preserving order, so a multi-job file is planned once.
+    for path in dict.fromkeys(c.file.path for c in targets):
+        pp.header(f"plan: {path.name}")
+        try:
+            nomad.validate(path)
+            nomad.plan(path)
+        except NomadBinaryError as exc:
+            pp.error(str(exc))
+            failures += 1
+    return 1 if failures else 0

nd/commands/run.py

@@ -0,0 +1,372 @@
+"""The ``nd run`` command: deploy job files and watch the rollout live."""
+
+from __future__ import annotations
+
+import asyncio
+import enum
+import time
+from dataclasses import dataclass, replace
+from typing import TYPE_CHECKING, Annotated
+
+import msgspec
+import typer
+from nclutils import pp
+
+from nd.binary import NomadBinary, NomadBinaryError
+from nd.commands._common import VerboseOption, configure_verbosity
+from nd.constants import DEPLOY_TIMEOUT_SECONDS, HEALTHY_ALLOC_STATUSES, POLL_INTERVAL_SECONDS
+from nd.jobfiles import candidates_for, discover_job_files, load_job_directories
+from nd.nomad import NomadClient, NomadConfig
+from nd.nomad.errors import NomadError
+from nd.targets import resolve_targets, select_candidates
+from nd.ui.alloc_rows import alloc_children
+from nd.ui.duration import summary_title
+from nd.ui.live_panel import PanelUpdate, run_rows
+from nd.ui.styles import OUTCOME_GLYPH
+
+if TYPE_CHECKING:
+    from nd.jobfiles import JobCandidate
+    from nd.nomad.models.deployment import Deployment
+    from nd.ui.alloc_rows import TaskLifecycle
+
+# Deployment statuses that mean the rollout is finished, one way or the other.
+_DEPLOY_SUCCESS = "successful"
+_DEPLOY_FAILURE = frozenset({"failed", "cancelled"})
+
+
+class DeployStatus(enum.StrEnum):
+    """The terminal outcome of deploying one job."""
+
+    DEPLOYED = "deployed"
+    FAILED = "failed"
+    TIMEOUT = "timeout"
+
+
+@dataclass(frozen=True)
+class DeployOutcome:
+    """The result of deploying one job, ready for summary rendering."""
+
+    name: str
+    status: DeployStatus
+    detail: str = ""
+    warnings: str = ""
+
+
+def deploy_phase(dep: Deployment) -> str:
+    """Summarize a deployment's progress as ``<status>: <healthy>/<desired> healthy``.
+
+    Aggregates counts across all task groups so the live panel shows a single
+    meaningful number rather than per-group noise.
+    """
+    healthy = sum(tg.healthy_allocs for tg in dep.task_groups.values())
+    desired = sum(tg.desired_total for tg in dep.task_groups.values())
+    return f"{dep.status}: {healthy}/{desired} healthy"
+
+
+def task_lifecycle(body: bytes) -> TaskLifecycle:
+    """Parse task lifecycle order and labels from a compiled job spec.
+
+    Tasks are ordered prestart, then main, then poststart/sidecar within each
+    group, so the panel shows them in the order Nomad runs them. Poststop tasks are
+    omitted because they only run when an allocation stops, not during a deploy.
+
+    Args:
+        body: The compiled ``{"Job": {...}}`` JSON from ``nomad job run -output``.
+
+    Returns:
+        A map of group name to ``{task name: (sort order, label)}``.
+    """
+    job = msgspec.json.decode(body).get("Job") or {}
+    lifecycle: TaskLifecycle = {}
+    for group in job.get("TaskGroups") or []:
+        tasks: dict[str, tuple[int, str]] = {}
+        for index, task in enumerate(group.get("Tasks") or []):
+            role = _task_role(task.get("Lifecycle"), index)
+            if role is not None:
+                tasks[task["Name"]] = role
+        lifecycle[group["Name"]] = tasks
+    return lifecycle
+
+
+def _task_role(lifecycle: dict[str, object] | None, index: int) -> tuple[int, str] | None:
+    """Return a task's (sort order, label) from its lifecycle block, or None to skip.
+
+    A task with no lifecycle block is a main task. Poststop tasks return None so
+    they are excluded from the deploy view.
+    """
+    if not lifecycle:
+        return (1_000 + index, "main")
+    hook = lifecycle.get("Hook")
+    if hook == "prestart":
+        return (index, "prestart")
+    if hook == "poststart":
+        return (2_000 + index, "sidecar" if lifecycle.get("Sidecar") else "poststart")
+    if hook == "poststop":
+        return None
+    return (1_000 + index, "main")
+
+
+_OUTCOME_ROW: dict[DeployStatus, tuple[str, str]] = {
+    DeployStatus.DEPLOYED: (OUTCOME_GLYPH["ok"], "deployed"),
+    DeployStatus.FAILED: (OUTCOME_GLYPH["fail"], "failed"),
+    DeployStatus.TIMEOUT: (OUTCOME_GLYPH["warn"], "still deploying"),
+}
+
+
+async def _running_job_names(client: NomadClient) -> set[str]:
+    """Return the names of jobs currently running in the cluster."""
+    jobs = await client.jobs.list()
+    return {j.name for j in jobs if j.status == "running"}
+
+
+# allow_interspersed_args lets options follow the positional JOB argument;
+# Typer groups disable that by default, which would parse flags as subcommands.
+app = typer.Typer(context_settings={"allow_interspersed_args": True})
+
+
+@app.callback(invoke_without_command=True)
+def run(
+    ctx: typer.Context,
+    job: Annotated[
+        str | None,
+        typer.Argument(
+            help="Job to run; matches any not-running job whose name starts with this. "
+            "Omit to pick from a list."
+        ),
+    ] = None,
+    detach: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option(
+            "--detach", "-d", help="Register the jobs and return without watching the rollout."
+        ),
+    ] = False,
+    dry_run: Annotated[  # noqa: FBT002
+        bool,
+        typer.Option("--dry-run", "-n", help="Resolve and validate without registering."),
+    ] = False,
+    verbose: VerboseOption = 0,
+) -> None:
+    """Deploy one or more not-yet-running job files and watch them roll out.
+
+    Only jobs that are not already running are offered; use plan to preview changes
+    to a running job. Each selected file is validated, registered, and watched live:
+    service jobs follow their deployment to success, while batch and system jobs
+    follow their allocations. Use --detach to register and return without watching.
+    """
+    configure_verbosity(ctx, verbose)
+    exit_code = asyncio.run(_run(job_arg=job, detach=detach, dry_run=dry_run))
+    if exit_code != 0:
+        raise typer.Exit(exit_code)
+
+
+async def _run(*, job_arg: str | None, detach: bool, dry_run: bool) -> int:  # noqa: PLR0911
+    """Resolve not-running candidates, validate, register, and watch the rollout.
+
+    Returns the exit code: 0 on clean success, 1 on any failure. With ``detach`` the
+    jobs are compiled and registered but the rollout is not watched.
+    """
+    files = discover_job_files(load_job_directories())
+    config = NomadConfig.resolve()
+    async with NomadClient.from_config(config) as client:
+        running = await _running_job_names(client)
+        candidates = candidates_for(files, exclude_names=running)
+        if not candidates:
+            pp.info("No deployable job files (all known jobs are already running).")
+            return 0
+
+        resolution = resolve_targets(candidates, job_arg, name_of=lambda c: c.name)
+        targets = await select_candidates(
+            resolution, "Select jobs to run", label_of=lambda c: f"{c.name}  [{c.file.path.name}]"
+        )
+        if targets is None:
+            return 0
+        if not targets:
+            pp.error(f"No not-running job file matching '{job_arg}'")
+            return 1
+
+        try:
+            nomad = NomadBinary.create(config)
+            # dict.fromkeys dedups so a multi-job file is validated once.
+            for path in dict.fromkeys(c.file.path for c in targets):
+                nomad.validate(path)
+        except NomadBinaryError as exc:
+            pp.error(str(exc))
+            return 1
+
+        if dry_run:
+            for c in targets:
+                pp.dryrun(f"would run {c.name} ({c.file.path})")
+            return 0
+
+        if detach:
+            return await _register_detached(client, targets, nomad)
+
+        outcomes = await _deploy_all(client, targets, nomad)
+
+    return 0 if all(o.status is DeployStatus.DEPLOYED for o in outcomes) else 1
+
+
+async def _register_detached(
+    client: NomadClient, targets: list[JobCandidate], nomad: NomadBinary
+) -> int:
+    """Compile and register every target concurrently, then return without watching.
+
+    Mirrors ``nomad job run -detach``: each job file is compiled to JSON and
+    registered, surfacing any register warnings, but the rollout is not polled. A
+    per-job compile or register failure is reported and does not abort the others.
+    Returns 0 only when every job registered successfully.
+    """
+
+    async def register_one(candidate: JobCandidate) -> tuple[str, str | None, str]:
+        try:
+            body = await asyncio.to_thread(nomad.compile_to_json, candidate.file.path)
+            resp = await client.jobs.register(body)
+        except (NomadBinaryError, NomadError) as exc:
+            return (candidate.name, str(exc), "")
+        return (candidate.name, None, resp.warnings)
+
+    results = await asyncio.gather(*(register_one(c) for c in targets))
+    registered = [name for name, err, _ in results if err is None]
+    if registered:
+        pp.success(f"Registered {len(registered)} job(s)", details=registered)
+    for name, err, warnings in results:
+        if err is not None:
+            pp.error(f"{name} failed to register", details=[err])
+        elif warnings:
+            pp.warning(f"{name}: {warnings}")
+    return 0 if all(err is None for _, err, _ in results) else 1
+
+
+async def _deploy_all(
+    client: NomadClient, targets: list[JobCandidate], nomad: NomadBinary
+) -> list[DeployOutcome]:
+    """Register and watch every target concurrently under one live panel.
+
+    Args:
+        client: Authenticated Nomad client.
+        targets: The job candidates to register and watch.
+        nomad: Configured `nomad` binary handle for the compile step.
+
+    Returns:
+        Ordered list of outcomes, one per target.
+    """
+    # Resolve node IDs to names once so every job's detail rows can show placement.
+    node_names = {node.id: node.name for node in await client.nodes.list()}
+
+    async def do_work(candidate: JobCandidate, update: PanelUpdate) -> DeployOutcome:
+        return await _deploy_one(
+            client, candidate, node_names=node_names, update=update, nomad=nomad
+        )
+
+    ordered = await run_rows(
+        targets,
+        do_work,
+        label_of=lambda c: c.name,
+        initial_phase="registering",
+        finish_of=lambda o: _OUTCOME_ROW[o.status],
+        running_title=f"Deploying {len(targets)} job(s)",
+        final_title=_final_title,
+    )
+
+    for o in ordered:
+        if o.status is DeployStatus.TIMEOUT:
+            pp.warning(f"{o.name}: {o.detail or 'still deploying'}")
+        elif o.status is DeployStatus.FAILED:
+            pp.error(f"{o.name} failed to deploy", details=[o.detail] if o.detail else None)
+        if o.warnings:
+            pp.warning(f"{o.name}: {o.warnings}")
+    return ordered
+
+
+def _final_title(outcomes: list[DeployOutcome], elapsed_seconds: float) -> str:
+    """Build the final panel title with deployed totals and elapsed seconds."""
+    ok = sum(1 for o in outcomes if o.status is DeployStatus.DEPLOYED)
+    return summary_title("Deployed", ok, len(outcomes), elapsed_seconds)
+
+
+async def _deploy_one(
+    client: NomadClient,
+    candidate: JobCandidate,
+    *,
+    node_names: dict[str, str],
+    update: PanelUpdate,
+    nomad: NomadBinary,
+) -> DeployOutcome:
+    """Compile, register, and watch one job to a terminal deploy state.
+
+    Service jobs are watched via their deployment; batch/system jobs (which create
+    no deployment) are watched via their allocations. Never raises: Nomad/binary
+    failures become a FAILED outcome so a sibling job's progress is unaffected.
+
+    Args:
+        client: Authenticated Nomad client.
+        candidate: The job file and name to deploy.
+        node_names: Map of node ID to node name for the per-allocation detail rows.
+        update: Callback to update the live panel phase text and detail rows.
+        nomad: Configured `nomad` binary handle for the compile step.
+
+    Returns:
+        The terminal outcome for this candidate.
+    """
+    try:
+        update("compiling")
+        # compile_to_json shells out to the nomad binary (blocking); run it off the
+        # event loop so sibling deploys keep making progress concurrently.
+        body = await asyncio.to_thread(nomad.compile_to_json, candidate.file.path)
+        lifecycle = task_lifecycle(body)
+        update("registering")
+        resp = await client.jobs.register(body)
+        outcome = await _watch(
+            client, candidate.name, node_names=node_names, lifecycle=lifecycle, update=update
+        )
+        # Attach any register warnings so the caller can surface them after the panel closes.
+        return replace(outcome, warnings=resp.warnings)
+    except (NomadBinaryError, NomadError) as exc:
+        return DeployOutcome(candidate.name, DeployStatus.FAILED, str(exc))
+
+
+async def _watch(
+    client: NomadClient,
+    job_id: str,
+    *,
+    node_names: dict[str, str],
+    lifecycle: TaskLifecycle,
+    update: PanelUpdate,
+) -> DeployOutcome:
+    """Poll a registered job until its deployment (or allocations) settle or time out.
+
+    Service jobs expose a deployment that tracks health; batch/system jobs have no
+    deployment so alloc statuses are used instead. Either way the job's allocations
+    are fetched each tick to show where each one is placed and its status. The poll
+    loop is bounded by a wall-clock deadline to avoid hanging on a stalled cluster.
+
+    Args:
+        client: Authenticated Nomad client.
+        job_id: The Nomad job ID to poll.
+        node_names: Map of node ID to node name for the per-allocation detail rows.
+        lifecycle: Task ordering and labels from the compiled job spec.
+        update: Callback to update the live panel phase text and detail rows.
+
+    Returns:
+        The terminal deploy outcome for this job.
+    """
+    deadline = time.monotonic() + DEPLOY_TIMEOUT_SECONDS
+    while True:
+        allocs = await client.jobs.allocations(job_id)
+        children = alloc_children(allocs, node_names, lifecycle)
+        deployments = await client.jobs.deployments(job_id)
+        if deployments:  # service job: follow the most-recent deployment
+            dep = await client.deployments.read(deployments[0].id)
+            if dep.status == _DEPLOY_SUCCESS:
+                return DeployOutcome(job_id, DeployStatus.DEPLOYED)
+            if dep.status in _DEPLOY_FAILURE:
+                return DeployOutcome(job_id, DeployStatus.FAILED, dep.status_description)
+            update(deploy_phase(dep), children)
+        else:  # batch/system job: follow allocations
+            running = sum(1 for a in allocs if a.client_status in HEALTHY_ALLOC_STATUSES)
+            if allocs and running == len(allocs):
+                return DeployOutcome(job_id, DeployStatus.DEPLOYED)
+            update(f"placing {running}/{len(allocs) or '?'} allocs", children)
+        if time.monotonic() >= deadline:
+            return DeployOutcome(job_id, DeployStatus.TIMEOUT, "deploy still in progress")
+        await asyncio.sleep(POLL_INTERVAL_SECONDS)