hyperun 0.2.1__tar.gz → 0.2.2__tar.gz

{hyperun-0.2.1 → hyperun-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperun
-Version: 0.2.1
+Version: 0.2.2
 Summary: Submit a GPU job and get results back. No kubectl, no cloud account.
 Author: DDPS Lab
 License-Expression: Apache-2.0

{hyperun-0.2.1 → hyperun-0.2.2}/hyperun/cli.py

@@ -50,6 +50,7 @@ from __future__ import annotations
 
 import argparse
 import base64
+import datetime
 import getpass
 import json
 import time
@@ -387,8 +388,13 @@ def build_parser() -> argparse.ArgumentParser:
     )
     watch.add_argument("job_id")
     watch.add_argument(
-        "--window", type=int, default=3600, metavar="SECONDS",
-        help="how far back to read (default 3600, max 86400)",
+        # default=None, not 3600, and the difference is load-bearing: it is how
+        # `cmd_watch` tells "the caller wants an hour" from "the caller said
+        # nothing", and only the second one may be widened to reach a finished
+        # job's readings. 604800 is the server's cap, not a number chosen here.
+        "--window", type=int, default=None, metavar="SECONDS",
+        help="how far back to read (default 3600, max 604800). A finished job "
+             "is widened automatically to reach its own readings.",
     )
     add_json_flag(watch)
 
@@ -1088,13 +1094,128 @@ def bar(percent: float, width: int = 20) -> str:
     return "#" * filled + "-" * (width - filled)
 
 
+# DDPSRUN-WATCH-TERMINAL. The three phases a job never leaves. They are the
+# controller's own words: the phase enum in api/v1alpha1/pacsjob_types.go
+# carries Pending / Starting / Running / Recovering / Succeeded / Failed /
+# Compared and nothing else, and the screen keeps the same three in
+# `const TERMINAL` (ui/app.js). Recovering is NOT here -- the machine was
+# reclaimed and the job is being restarted, so the run is still going.
+TERMINAL_PHASES = ("Succeeded", "Failed", "Compared")
+
+# How far back `watch` reads when the caller says nothing, and the furthest the
+# server will let it reach. The cap is the server's own: `window_seconds` on
+# `GET /v1/jobs/{id}/metrics` is declared `ge=60, le=604800`
+# (server/ddpsrun_server/main.py). The `--window` help said 86400 here, which was
+# simply wrong -- one day, against the server's seven.
+DEFAULT_WATCH_WINDOW = 3600
+MAX_WATCH_WINDOW = 604800
+
+
+def _window_reaching_back_to_start(job: dict[str, Any]) -> int:
+    """How wide a window must be to contain a FINISHED job's readings.
+
+    ★ WHY THIS EXISTS. Nothing is stored: `watch` re-reads the job's own log, and
+    the window is measured back from NOW. A running job's readings are therefore
+    always inside the default hour. A job that finished five hours ago has NONE
+    of its readings in the last hour, so `hyperun watch` on it printed the phase
+    and no numbers -- and the phase is the one thing the reader already knew.
+
+    The fix is not a larger default, which would make every `watch` on a running
+    job re-read a week of log. It is to widen only when the job is over AND the
+    caller did not pick a window.
+
+    Args:
+        job: the body of `GET /v1/jobs/{id}`.
+
+    Returns:
+        Seconds, or 0 when the job is not finished or carries no start time, in
+        which case the caller leaves the window as it was.
+    """
+    if job.get("phase") not in TERMINAL_PHASES:
+        return 0
+    started = job.get("started_at") or job.get("created_at")
+    if not started:
+        return 0
+    try:
+        began = datetime.datetime.fromisoformat(started.replace("Z", "+00:00"))
+    except ValueError:
+        return 0
+    elapsed = (datetime.datetime.now(datetime.timezone.utc) - began).total_seconds()
+    # A minute of headroom so the FIRST reading falls inside the window instead
+    # of exactly on its edge, and the server's own cap over the top.
+    return min(MAX_WATCH_WINDOW, max(60, int(elapsed) + 60))
+
+
 def cmd_watch(args: argparse.Namespace) -> int:
-    """Print a job's GPU usage and how far the training has got."""
-    result = client_from_config().metrics(args.job_id, args.window)
+    """Print a job's GPU usage and how far the training has got.
+
+    ONE BLOCK PER CARD, because a job can rent several. baseline-c rents four
+    A100s in one pod, and until today this printed `latest_gpu`, which is card 0
+    alone -- three quarters of a $44 run was invisible from the CLI. The screen
+    had the same defect and was fixed on 2026-09-08. `cards` is the server's
+    per-card answer (CardMetricsView in server/ddpsrun_server/models.py); a
+    server too old to send it answers with an empty list, and then the
+    single-card block below is the whole story, exactly as before.
+
+    WHY THIS ASKS FOR THE PHASE AS WELL AS THE METRICS. A finished job's last
+    reading is the idle card in the seconds before teardown -- 0%, 0 MiB -- so
+    printing it under "GPU util" tells a reader the run was idle when it was
+    not. Leading with the peak instead requires knowing the job is over, and
+    THE METRICS ANSWER DOES NOT SAY: MetricsResponse carries latest_gpu,
+    gpu_series, peak_gpu, the two utilisation figures, sample_count, cards,
+    progress, window_seconds and note, and no phase at all
+    (server/ddpsrun_server/models.py). Guessing from the readings fails in both
+    directions -- a job whose framework has not allocated yet also reads 0 MiB,
+    and a job killed mid-step leaves a final reading that is nowhere near 0 --
+    so this asks the server instead.
+
+    WHAT THAT COSTS: one extra `GET /v1/jobs/{id}` per typed `watch`. `watch`
+    prints once and exits rather than polling, so it is one extra request per
+    command, not one per second, and `--json` pays it only when the first window
+    came back empty. The screen pays the same price -- it reads the job and hands
+    it to drawMetrics (ui/app.js).
+    """
+    client = client_from_config()
+    window = DEFAULT_WATCH_WINDOW if args.window is None else args.window
+    result = client.metrics(args.job_id, window)
+
+    # ★ AN EMPTY ANSWER ON A FINISHED JOB IS THE WINDOW, NOT THE JOB. The window
+    # is measured back from now, so a run that ended hours ago has every reading
+    # outside the default hour and `watch` printed nothing. Widening is done
+    # ONLY here: the caller did not choose a window, and the first one came back
+    # with no readings at all. A running job never reaches this branch, so the
+    # usual `watch` is still exactly one metrics request.
+    #
+    # "No readings" means BOTH are empty. `cards` is the per-card answer and
+    # `latest_gpu` is card 0; a multi-card job fills `cards`, while a server too
+    # old to send it fills only `latest_gpu`. Testing one alone would make
+    # `watch --json` on a four-card job buy a second request it does not need.
+    job: dict[str, Any] | None = None
+    if args.window is None and not result.get("latest_gpu") and not result.get("cards"):
+        try:
+            job = client.status(args.job_id)
+        except ServerError:
+            job = None
+        wider = _window_reaching_back_to_start(job) if job else 0
+        if wider > window:
+            result = client.metrics(args.job_id, wider)
+            window = wider
+
     if args.json:
+        # The server's answer, verbatim. It already contains `cards`, so nothing
+        # here needs the phase.
         print(json.dumps(result, indent=2, ensure_ascii=False))
         return EXIT_OK
 
+    # A phase lookup that fails must not lose the GPU numbers this command
+    # exists to print, so it falls back to the running layout. The likely cause
+    # is the job being deleted between the two calls, and the running layout is
+    # the behaviour this command had until today rather than a wrong number.
+    try:
+        done = (job or client.status(args.job_id)).get("phase") in TERMINAL_PHASES
+    except ServerError:
+        done = False
+
     progress = result.get("progress")
     if progress:
         print(f"  training       {bar(progress['percent'])}  "
@@ -1109,12 +1230,40 @@ def cmd_watch(args: argparse.Namespace) -> int:
 
     gpu = result.get("latest_gpu")
     if gpu:
-        print(f"  GPU util       {bar(gpu['utilization_percent'])}  "
-              f"{gpu['utilization_percent']}%")
-        print(f"  GPU memory     {bar(gpu['memory_percent'])}  "
-              f"{gpu['memory_used_mib']:,} / {gpu['memory_total_mib']:,} MiB "
-              f"({gpu['memory_percent']}%)")
-        print(f"  temp, power    {gpu['temperature_c']} C, {gpu['power_w']} W")
+        # The reading with the most MEMORY in use. A server too old to send it
+        # leaves the last reading as the only one there is, which is what the
+        # screen falls back to as well.
+        peak = result.get("peak_gpu") or gpu
+        if done:
+            # A FINISHED RUN LEADS WITH THE PEAK. Memory is what kills a run, so
+            # the peak is the number a post-mortem came here for; the last
+            # reading is kept, because it is still evidence, but it is named for
+            # what it is instead of being labelled "GPU util". The screen was
+            # changed to this shape on 2026-09-07 after the 0%, 0 MiB readout
+            # was reported as the panel being broken.
+            print(f"  peak memory    {bar(peak['memory_percent'])}  "
+                  f"{peak['memory_used_mib']:,} / {peak['memory_total_mib']:,} MiB "
+                  f"({peak['memory_percent']}%)")
+            # ★ `peak_utilization_percent`, NOT `peak['utilization_percent']`,
+            # and the difference is the whole defect. `peak` is the sample with
+            # the most memory in it and its utilisation is whatever the card
+            # happened to be doing at that instant. On job-66b46719b854 all four
+            # A100s reached 77,631 MiB and the utilisation inside those four
+            # samples read 0, 3, 1 and 95 -- while every one of those cards
+            # actually peaked at 99 or 100 and averaged between 37.8 and 84.6.
+            # Reported on the screen 2026-09-11; the CLI never printed either
+            # figure, so it is being added correct rather than fixed.
+            print(f"  peak util      {_percent(result.get('peak_utilization_percent'))}")
+            print(f"  avg util       {_percent(result.get('avg_utilization_percent'))}")
+            print(f"  last reading   {gpu['utilization_percent']}%, "
+                  f"{gpu['memory_used_mib']:,} MiB  (run ended)")
+        else:
+            print(f"  GPU util       {bar(gpu['utilization_percent'])}  "
+                  f"{gpu['utilization_percent']}%")
+            print(f"  GPU memory     {bar(gpu['memory_percent'])}  "
+                  f"{gpu['memory_used_mib']:,} / {gpu['memory_total_mib']:,} MiB "
+                  f"({gpu['memory_percent']}%)")
+            print(f"  temp, power    {gpu['temperature_c']} C, {gpu['power_w']} W")
         # `sample_count`, not len(gpu_series): the server thins the series to at
         # most 400 points so a chart can draw it, and job-66b46719b854 printed
         # 785 readings per card while this line said 393. The screen had the same
@@ -1122,12 +1271,61 @@ def cmd_watch(args: argparse.Namespace) -> int:
         # falls back to the thinned length, which is the only number it has.
         taken = result.get("sample_count") or len(result.get("gpu_series", []))
         print(f"  samples        {taken}, last {result['window_seconds']}s")
+        print_card_table(result.get("cards") or [])
 
     if result.get("note"):
         print(f"  note           {result['note']}")
     return EXIT_OK
 
 
+def _percent(value: float | int | None) -> str:
+    """One utilisation figure, or a dash when the server did not send it.
+
+    The two utilisation fields arrived on 2026-09-10 and a server older than
+    that answers with them absent. A blank line reads as 0, which is the very
+    thing the peak figures exist to stop being claimed.
+    """
+    return "-" if value is None else f"{value}%"
+
+
+def print_card_table(cards: list[dict[str, Any]]) -> None:
+    """One row per GPU card, printed only when the job rented more than one.
+
+    WHY THIS EXISTS. Everything above describes card 0 -- `latest_gpu` and
+    `peak_gpu` are the lowest-indexed card's readings, unchanged since before
+    `cards` existed. job-66b46719b854 is four A100s in one pod, and reading its
+    CLI output left three of them invisible.
+
+    WHY ONLY ABOVE ONE CARD. With a single card these four numbers repeat the
+    block above it word for word, so the table is drawn on the same condition
+    the screen draws its own (`cards.length > 1` in ui/app.js).
+
+    Args:
+        cards: the response's `cards`, each a CardMetricsView -- `gpu_index`,
+            `series`, `latest`, `peak`, `avg_utilization_percent`,
+            `peak_utilization_percent`, `sample_count`.
+    """
+    if len(cards) < 2:
+        return
+    print(f"  {'card':<8}{'peak memory':>27}{'peak util':>11}"
+          f"{'avg util':>10}{'samples':>9}")
+    for card in cards:
+        # `peak` is this card's highest-MEMORY reading; falling back to its last
+        # one matches the screen's table and keeps a row rather than dropping a
+        # card that only ever printed once.
+        sample = card.get("peak") or card.get("latest") or {}
+        memory = "-" if not sample else (
+            f"{sample['memory_used_mib']:,} / {sample['memory_total_mib']:,} MiB "
+            f"({sample['memory_percent']:.0f}%)")
+        # Same two corrections as the block above: the card's own highest
+        # utilisation rather than the utilisation inside its memory peak, and
+        # the readings it printed rather than the points left after thinning.
+        taken = card.get("sample_count") or len(card.get("series") or [])
+        print(f"  {('GPU ' + str(card['gpu_index'])):<8}{memory:>27}"
+              f"{_percent(card.get('peak_utilization_percent')):>11}"
+              f"{_percent(card.get('avg_utilization_percent')):>10}{taken:>9}")
+
+
 def cmd_stats(args: argparse.Namespace) -> int:
     """Print this caller's team figures."""
     result = client_from_config().stats()

{hyperun-0.2.1 → hyperun-0.2.2}/hyperun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperun
-Version: 0.2.1
+Version: 0.2.2
 Summary: Submit a GPU job and get results back. No kubectl, no cloud account.
 Author: DDPS Lab
 License-Expression: Apache-2.0

{hyperun-0.2.1 → hyperun-0.2.2}/pyproject.toml

@@ -34,7 +34,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hyperun"
-version = "0.2.1"
+version = "0.2.2"
 description = "Submit a GPU job and get results back. No kubectl, no cloud account."
 requires-python = ">=3.9"
 dependencies = ["requests>=2.31", "PyYAML>=6.0"]

{hyperun-0.2.1 → hyperun-0.2.2}/tests/test_cli.py

@@ -38,6 +38,15 @@ class FakeClient:
         self.estimate_result = {}
         self.validate_result = {"ok": True, "findings": [], "not_checked": []}
         self.metrics_result = {"window_seconds": 3600, "gpu_series": [], "note": ""}
+        # Every window `watch` asked for, in order. The widening of a finished
+        # job's window is invisible in the printed output -- the numbers look the
+        # same whichever window found them -- so the only way to test it is to
+        # record what was asked.
+        self.metrics_windows: list[int] = []
+        # What to answer once the window is wider than the default hour. None
+        # means "the same answer whatever the window", which is what every test
+        # written before the widening existed expects.
+        self.metrics_wide_result = None
         self.stats_result = {"team": "", "members": [], "jobs": 0, "gpu_hours": 0.0,
                              "cost_usd": 0.0, "unpriced_jobs": 0, "note": ""}
         self.secrets_result = {"names": ["GITHUB_PAT", "HF_TOKEN"],
@@ -55,6 +64,9 @@ class FakeClient:
         return self.validate_result
 
     def metrics(self, job_id, window_seconds=3600):
+        self.metrics_windows.append(window_seconds)
+        if self.metrics_wide_result is not None and window_seconds > 3600:
+            return self.metrics_wide_result
         return self.metrics_result
 
     def stats(self):
@@ -475,6 +487,265 @@ def test_watch_prints_progress_and_gpu(fake, capsys):
     assert "samples        120" in printed
 
 
+def gpu_sample(util, used, total=81920, index=0):
+    """One reading, for the card tests below."""
+    return {"utilization_percent": util, "memory_used_mib": used,
+            "memory_total_mib": total, "memory_percent": round(100 * used / total, 1),
+            "temperature_c": 62, "power_w": 310.0, "gpu_index": index}
+
+
+def four_a100_cards():
+    """job-66b46719b854 as the server reports it: four A100s in one pod.
+
+    The numbers are that run's: every card peaked at 77,631 of 81,920 MiB, and
+    the utilisation INSIDE those four highest-memory samples read 0, 3, 1 and 95
+    while the cards themselves peaked at 100, 99, 100 and 99. That gap is the
+    defect the screen carried until 2026-09-11, so the fixture keeps it.
+    """
+    peaks = [(0, 100, 84.6), (3, 99, 37.8), (1, 100, 79.2), (95, 99, 81.3)]
+    return [
+        {"gpu_index": i,
+         "series": [{}] * 393,
+         "latest": gpu_sample(0, 0, index=i),
+         "peak": gpu_sample(at_peak, 77631, index=i),
+         "avg_utilization_percent": avg,
+         "peak_utilization_percent": real_peak,
+         "sample_count": 785}
+        for i, (at_peak, real_peak, avg) in enumerate(peaks)
+    ]
+
+
+def test_every_card_is_printed_not_only_the_first(fake, capsys):
+    # ★ THE DEFECT. `latest_gpu` is card 0, so a four-A100 job printed one card
+    # and three quarters of a $44 run was invisible from the CLI. The screen was
+    # fixed on 2026-09-08 and this surface was not.
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200),
+        "peak_gpu": gpu_sample(0, 77631),
+        "gpu_series": [{}] * 393,
+        "sample_count": 785,
+        "peak_utilization_percent": 100,
+        "avg_utilization_percent": 84.6,
+        "cards": four_a100_cards(),
+        "window_seconds": 604800, "note": "",
+    }
+    assert run(["watch", "job-66b46719b854"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    # `GPU 0` is a table row; `GPU util` and `GPU memory` are the headline above
+    # it, so the card index is what tells the two apart.
+    rows = {line.split()[1]: line.split()
+            for line in printed.splitlines()
+            if line.startswith("  GPU ") and line.split()[1].isdigit()}
+    assert sorted(rows) == ["0", "1", "2", "3"]
+
+    # A row reads: GPU <n> <used> / <total> MiB (<pct>%) <peak util> <avg> <n>
+    # so the last three fields are the three that were wrong or missing.
+    expected = {"0": ("100%", "84.6%"), "1": ("99%", "37.8%"),
+                "2": ("100%", "79.2%"), "3": ("99%", "81.3%")}
+    for index, (peak_util, avg_util) in expected.items():
+        fields = rows[index]
+        # Each card's own peak memory, not card 0's repeated four times.
+        assert fields[2:7] == ["77,631", "/", "81,920", "MiB", "(95%)"]
+        # ★ `peak_utilization_percent`, NOT the utilisation inside the memory
+        # peak. The two differ on this very run: the four highest-memory samples
+        # read 0%, 3%, 1% and 95% while the cards peaked at 100, 99, 100 and 99.
+        assert fields[7] == peak_util
+        assert fields[8] == avg_util
+        # The readings this card printed, not the 393 points left after the
+        # server thinned the series for the chart.
+        assert fields[9] == "785"
+
+
+def test_one_card_prints_no_table_because_it_would_repeat_the_block(fake, capsys):
+    # With a single card the table's four numbers are the same four printed
+    # directly above it, so it is drawn on the same condition the screen uses.
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "gpu_series": [{}] * 120,
+        "cards": [{"gpu_index": 0, "series": [{}] * 120,
+                   "latest": gpu_sample(94, 38200, total=45440),
+                   "peak": gpu_sample(94, 38200, total=45440),
+                   "avg_utilization_percent": 90.0,
+                   "peak_utilization_percent": 99,
+                   "sample_count": 120}],
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "peak memory" not in printed      # the table's header row
+    assert "38,200 / 45,440 MiB" in printed
+
+
+def test_a_finished_job_leads_with_the_peak_not_the_idle_last_reading(fake, capsys):
+    # ★ THE DEFECT. A finished job's last reading is the idle card in the
+    # seconds before teardown -- 0%, 0 MiB -- and printing it as "GPU util" says
+    # the run was idle. What a post-mortem asks for is the peak, because running
+    # out of memory is what kills a run. The screen was changed to this shape on
+    # 2026-09-07.
+    fake.status_result = {"job_id": "job-66b46719b854", "name": "baseline-c",
+                          "phase": "Succeeded"}
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(0, 0),
+        "peak_gpu": gpu_sample(0, 77631),
+        "gpu_series": [{}] * 393,
+        "sample_count": 785,
+        "peak_utilization_percent": 100,
+        "avg_utilization_percent": 84.6,
+        "window_seconds": 604800, "note": "",
+    }
+    assert run(["watch", "job-66b46719b854"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "peak memory" in printed
+    assert "77,631 / 81,920 MiB" in printed
+    # The peak of the CARD, not the utilisation inside the highest-memory
+    # sample, which on this run was 0 while the card reached 100.
+    assert "peak util      100%" in printed
+    assert "avg util       84.6%" in printed
+    # The last reading is still shown, and named for what it is.
+    assert "last reading   0%, 0 MiB  (run ended)" in printed
+    # The running job's labels are gone: leaving "GPU util  0%" on the screen is
+    # exactly the claim this fix removes.
+    assert "GPU util" not in printed
+
+
+def test_a_running_job_keeps_the_live_readings_as_the_headline(fake, capsys):
+    # The mirror of the test above. `peak_gpu` being present is not on its own a
+    # finished job -- the server sends it for a running one too -- so the phase
+    # is what decides, and a Running job still leads with what the card is doing
+    # NOW.
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "peak_gpu": gpu_sample(99, 40000, total=45440),
+        "gpu_series": [{}] * 120,
+        "peak_utilization_percent": 99,
+        "avg_utilization_percent": 90.0,
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "GPU util" in printed
+    assert "38,200 / 45,440 MiB" in printed
+    assert "last reading" not in printed
+
+
+def test_a_phase_lookup_that_fails_still_prints_the_gpu_numbers(fake, capsys):
+    # The numbers are what the command exists for, so a job deleted between the
+    # metrics call and the phase call falls back to the running layout rather
+    # than exiting with nothing printed.
+    def refuse(job_id):
+        raise ServerError("404: no such job")
+
+    fake.status = refuse
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "gpu_series": [{}] * 120,
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert "38,200 / 45,440 MiB" in capsys.readouterr().out
+
+
+def _hours_ago(hours):
+    """An RFC 3339 timestamp that many hours in the past, as the server writes it."""
+    import datetime as _dt
+    t = _dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=hours)
+    return t.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def test_watch_widens_the_window_to_reach_a_finished_jobs_readings(fake, capsys):
+    # ★ THE DEFECT: nothing is stored, so `watch` re-reads the log and the window
+    # is measured back from NOW. A run that ended five hours ago has every
+    # reading outside the default hour, and `watch` printed the phase and no
+    # numbers -- the phase being the one thing the reader already knew.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "name": "bank-exp2",
+                          "phase": "Succeeded", "started_at": _hours_ago(9),
+                          "finished_at": _hours_ago(5)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+    fake.metrics_wide_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "peak_gpu": gpu_sample(94, 38200, total=45440),
+        "peak_utilization_percent": 94,
+        "gpu_series": [{}] * 120, "window_seconds": 36000, "note": "",
+    }
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert len(fake.metrics_windows) == 2, "the empty first window was not retried"
+    assert fake.metrics_windows[0] == 3600
+    # Nine hours of run plus a minute of headroom, so the FIRST reading is inside
+    # the window rather than exactly on its edge.
+    assert fake.metrics_windows[1] >= 9 * 3600
+    assert "38,200 / 45,440 MiB" in capsys.readouterr().out
+
+
+def test_a_window_the_caller_chose_is_never_widened(fake):
+    # An explicit --window is an instruction, not a default. Widening it would
+    # answer a question the caller did not ask, and the answer would be a bigger
+    # read of the log than they wanted.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Succeeded",
+                          "started_at": _hours_ago(9)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07", "--window", "600"]) == cli.EXIT_OK
+    assert fake.metrics_windows == [600]
+
+
+def test_a_running_job_is_not_widened_however_quiet_it_is(fake):
+    # A running job's readings ARE inside the last hour, so an empty answer means
+    # the job has printed nothing yet. Re-reading a week of its log would buy a
+    # second empty answer and a much larger response.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Running",
+                          "started_at": _hours_ago(9)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert fake.metrics_windows == [3600]
+
+
+def test_the_widened_window_stops_at_the_servers_own_cap(fake):
+    # `window_seconds` is declared ge=60, le=604800 on the metrics route. Asking
+    # for more is a 422, so a job that started a month ago must be clamped here
+    # rather than turned into an error the reader cannot act on.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Failed",
+                          "started_at": _hours_ago(24 * 30)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert fake.metrics_windows[1] == cli.MAX_WATCH_WINDOW == 604800
+
+
+def test_a_finished_job_with_no_start_time_leaves_the_window_alone(fake):
+    # Nothing to compute a width from. Widening to the cap "just in case" would
+    # read seven days of log off the back of a missing field.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Succeeded"}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert fake.metrics_windows == [3600]
+
+
+def test_the_window_help_quotes_the_servers_real_cap(capsys):
+    # It said 86400 and the server accepts 604800, so the help talked a reader
+    # out of a window the server would have answered.
+    with pytest.raises(SystemExit):
+        cli.main(["watch", "--help"])
+    printed = capsys.readouterr().out
+    assert "604800" in printed
+    assert "86400" not in printed
+
+
+def test_watch_json_prints_the_cards_and_asks_for_no_phase(fake, capsys):
+    # --json is the server's answer verbatim, so the second request would buy
+    # nothing. It is one GET per typed `watch` and worth not making.
+    def refuse(job_id):
+        raise AssertionError("--json must not ask for the phase")
+
+    fake.status = refuse
+    fake.metrics_result = {"cards": four_a100_cards(), "gpu_series": [],
+                           "window_seconds": 3600, "note": ""}
+    assert run(["watch", "job-66b46719b854", "--json"]) == cli.EXIT_OK
+    assert len(json.loads(capsys.readouterr().out)["cards"]) == 4
+
+
 def test_samples_counts_the_readings_taken_not_the_chart_points(fake, capsys):
     # ★ THE SAME DEFECT THE SCREEN HAD. The server thins the series to at most
     # 400 points so a chart can draw it; job-66b46719b854 printed 785 readings