hyperun 0.2.0__tar.gz → 0.2.2__tar.gz

{hyperun-0.2.0 → hyperun-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperun
-Version: 0.2.0
+Version: 0.2.2
 Summary: Submit a GPU job and get results back. No kubectl, no cloud account.
 Author: DDPS Lab
 License-Expression: Apache-2.0

{hyperun-0.2.0 → hyperun-0.2.2}/hyperun/cli.py

@@ -50,6 +50,7 @@ from __future__ import annotations
 
 import argparse
 import base64
+import datetime
 import getpass
 import json
 import time
@@ -365,9 +366,10 @@ def build_parser() -> argparse.ArgumentParser:
         "through the job's driver pod into the workload container on the "
         "rented machine and brings the exit code back like ssh. It is NOT a "
         "TTY — no vim, no top, about 25 seconds per command — because the "
-        "server is a Lambda and cannot hold a terminal open. AWS and GCP "
-        "machine rentals only: a RunPod job is a rented container with no "
-        "machine behind it, and the relay refuses it. ★ PUT OPTIONS BEFORE THE "
+        "server is a Lambda and cannot hold a terminal open. AWS, GCP and "
+        "Shadeform machine rentals only: a RunPod job is a rented container "
+        "with no machine behind it, so there is no cluster to attach to and "
+        "the relay refuses it. ★ PUT OPTIONS BEFORE THE "
         "JOB ID -- everything after it is sent to the workload as-is, so "
         "`shell job-x --slot 2` asks pod 0 and passes `--slot 2` to the shell. "
         "That is refused rather than obeyed.",
@@ -386,8 +388,13 @@ def build_parser() -> argparse.ArgumentParser:
     )
     watch.add_argument("job_id")
     watch.add_argument(
-        "--window", type=int, default=3600, metavar="SECONDS",
-        help="how far back to read (default 3600, max 86400)",
+        # default=None, not 3600, and the difference is load-bearing: it is how
+        # `cmd_watch` tells "the caller wants an hour" from "the caller said
+        # nothing", and only the second one may be widened to reach a finished
+        # job's readings. 604800 is the server's cap, not a number chosen here.
+        "--window", type=int, default=None, metavar="SECONDS",
+        help="how far back to read (default 3600, max 604800). A finished job "
+             "is widened automatically to reach its own readings.",
     )
     add_json_flag(watch)
 
@@ -826,11 +833,26 @@ def cmd_shell(args: argparse.Namespace) -> int:
     pod onto the rented machine and returns the exit code like ssh. With a
     trailing `-- command` it runs once and exits with that code; without one
     it prompts, which FEELS like a slow shell and is honestly a request loop.
+
+    ★ THE PROMPT USES A SESSION AND THE ONE-SHOT FORM DOES NOT, since
+    2026-09-10. The prompt types into a shell that stays running in the driver
+    pod, so `cd`, an exported variable and an activated venv survive from one
+    line to the next -- which is what a person at a prompt assumes and what
+    every earlier version quietly did not do. A `-- command` invocation is a
+    script's shape and gets the stateless form: it wants no history and should
+    not leave a session behind for the idle timer to close.
     """
     client = client_from_config()
-
-    def run_once(line: str) -> int:
-        answer = client.exec_in_job(args.job, line, slot=args.slot)
+    # The output sequence the session has already shown us. It is here rather
+    # than inside run_once because it has to survive between lines, which is the
+    # whole point.
+    state = {"seq": 0}
+
+    def run_once(line: str, session: bool = False) -> int:
+        answer = client.exec_in_job(args.job, line, slot=args.slot,
+                                    session=session, seq=state["seq"])
+        if session:
+            state["seq"] = int(answer.get("seq", state["seq"]))
         output = answer.get("output") or ""
         if output:
             print(output, end="" if output.endswith("\n") else "\n")
@@ -873,7 +895,8 @@ def cmd_shell(args: argparse.Namespace) -> int:
         return run_once(" ".join(words))
 
     print(
-        f"one command per line, ~25s each; 'exit' or Ctrl-D leaves. Not a TTY.",
+        f"one command per line, ~25s each; 'exit' or Ctrl-D leaves. Not a TTY, "
+        f"but `cd` and exported variables DO survive between lines.",
         file=sys.stderr,
     )
     while True:
@@ -888,10 +911,29 @@ def cmd_shell(args: argparse.Namespace) -> int:
         if line in ("exit", "quit"):
             return EXIT_OK
         try:
-            code = run_once(line)
+            code = run_once(line, session=True)
             if code:
                 print(f"(exit {code})", file=sys.stderr)
         except ServerError as exc:
+            # A SESSION THAT IS GONE IS THE ONE ERROR WORTH ACTING ON. The
+            # server answers 409 when the driver pod has no session for this
+            # slot -- it timed out after ten minutes unread, or the workload
+            # restarted. Reopening is right there and only there: everywhere
+            # else a new shell would silently lose the `cd` the person is
+            # relying on, which is the failure this whole feature exists to
+            # end. The sequence resets with it, because the new shell's
+            # output starts from nothing.
+            if "open one first" in str(exc) or "no session" in str(exc):
+                state["seq"] = 0
+                print("(the session had closed; reopening — `cd` and variables "
+                      "from before are gone)", file=sys.stderr)
+                try:
+                    code = run_once(line, session=True)
+                    if code:
+                        print(f"(exit {code})", file=sys.stderr)
+                    continue
+                except ServerError as retry_exc:
+                    exc = retry_exc
             # One failed command must not end the session: say what the server
             # said and keep the prompt.
             print(f"error: {exc}", file=sys.stderr)
@@ -1052,13 +1094,128 @@ def bar(percent: float, width: int = 20) -> str:
     return "#" * filled + "-" * (width - filled)
 
 
+# DDPSRUN-WATCH-TERMINAL. The three phases a job never leaves. They are the
+# controller's own words: the phase enum in api/v1alpha1/pacsjob_types.go
+# carries Pending / Starting / Running / Recovering / Succeeded / Failed /
+# Compared and nothing else, and the screen keeps the same three in
+# `const TERMINAL` (ui/app.js). Recovering is NOT here -- the machine was
+# reclaimed and the job is being restarted, so the run is still going.
+TERMINAL_PHASES = ("Succeeded", "Failed", "Compared")
+
+# How far back `watch` reads when the caller says nothing, and the furthest the
+# server will let it reach. The cap is the server's own: `window_seconds` on
+# `GET /v1/jobs/{id}/metrics` is declared `ge=60, le=604800`
+# (server/ddpsrun_server/main.py). The `--window` help said 86400 here, which was
+# simply wrong -- one day, against the server's seven.
+DEFAULT_WATCH_WINDOW = 3600
+MAX_WATCH_WINDOW = 604800
+
+
+def _window_reaching_back_to_start(job: dict[str, Any]) -> int:
+    """How wide a window must be to contain a FINISHED job's readings.
+
+    ★ WHY THIS EXISTS. Nothing is stored: `watch` re-reads the job's own log, and
+    the window is measured back from NOW. A running job's readings are therefore
+    always inside the default hour. A job that finished five hours ago has NONE
+    of its readings in the last hour, so `hyperun watch` on it printed the phase
+    and no numbers -- and the phase is the one thing the reader already knew.
+
+    The fix is not a larger default, which would make every `watch` on a running
+    job re-read a week of log. It is to widen only when the job is over AND the
+    caller did not pick a window.
+
+    Args:
+        job: the body of `GET /v1/jobs/{id}`.
+
+    Returns:
+        Seconds, or 0 when the job is not finished or carries no start time, in
+        which case the caller leaves the window as it was.
+    """
+    if job.get("phase") not in TERMINAL_PHASES:
+        return 0
+    started = job.get("started_at") or job.get("created_at")
+    if not started:
+        return 0
+    try:
+        began = datetime.datetime.fromisoformat(started.replace("Z", "+00:00"))
+    except ValueError:
+        return 0
+    elapsed = (datetime.datetime.now(datetime.timezone.utc) - began).total_seconds()
+    # A minute of headroom so the FIRST reading falls inside the window instead
+    # of exactly on its edge, and the server's own cap over the top.
+    return min(MAX_WATCH_WINDOW, max(60, int(elapsed) + 60))
+
+
 def cmd_watch(args: argparse.Namespace) -> int:
-    """Print a job's GPU usage and how far the training has got."""
-    result = client_from_config().metrics(args.job_id, args.window)
+    """Print a job's GPU usage and how far the training has got.
+
+    ONE BLOCK PER CARD, because a job can rent several. baseline-c rents four
+    A100s in one pod, and until today this printed `latest_gpu`, which is card 0
+    alone -- three quarters of a $44 run was invisible from the CLI. The screen
+    had the same defect and was fixed on 2026-09-08. `cards` is the server's
+    per-card answer (CardMetricsView in server/ddpsrun_server/models.py); a
+    server too old to send it answers with an empty list, and then the
+    single-card block below is the whole story, exactly as before.
+
+    WHY THIS ASKS FOR THE PHASE AS WELL AS THE METRICS. A finished job's last
+    reading is the idle card in the seconds before teardown -- 0%, 0 MiB -- so
+    printing it under "GPU util" tells a reader the run was idle when it was
+    not. Leading with the peak instead requires knowing the job is over, and
+    THE METRICS ANSWER DOES NOT SAY: MetricsResponse carries latest_gpu,
+    gpu_series, peak_gpu, the two utilisation figures, sample_count, cards,
+    progress, window_seconds and note, and no phase at all
+    (server/ddpsrun_server/models.py). Guessing from the readings fails in both
+    directions -- a job whose framework has not allocated yet also reads 0 MiB,
+    and a job killed mid-step leaves a final reading that is nowhere near 0 --
+    so this asks the server instead.
+
+    WHAT THAT COSTS: one extra `GET /v1/jobs/{id}` per typed `watch`. `watch`
+    prints once and exits rather than polling, so it is one extra request per
+    command, not one per second, and `--json` pays it only when the first window
+    came back empty. The screen pays the same price -- it reads the job and hands
+    it to drawMetrics (ui/app.js).
+    """
+    client = client_from_config()
+    window = DEFAULT_WATCH_WINDOW if args.window is None else args.window
+    result = client.metrics(args.job_id, window)
+
+    # ★ AN EMPTY ANSWER ON A FINISHED JOB IS THE WINDOW, NOT THE JOB. The window
+    # is measured back from now, so a run that ended hours ago has every reading
+    # outside the default hour and `watch` printed nothing. Widening is done
+    # ONLY here: the caller did not choose a window, and the first one came back
+    # with no readings at all. A running job never reaches this branch, so the
+    # usual `watch` is still exactly one metrics request.
+    #
+    # "No readings" means BOTH are empty. `cards` is the per-card answer and
+    # `latest_gpu` is card 0; a multi-card job fills `cards`, while a server too
+    # old to send it fills only `latest_gpu`. Testing one alone would make
+    # `watch --json` on a four-card job buy a second request it does not need.
+    job: dict[str, Any] | None = None
+    if args.window is None and not result.get("latest_gpu") and not result.get("cards"):
+        try:
+            job = client.status(args.job_id)
+        except ServerError:
+            job = None
+        wider = _window_reaching_back_to_start(job) if job else 0
+        if wider > window:
+            result = client.metrics(args.job_id, wider)
+            window = wider
+
     if args.json:
+        # The server's answer, verbatim. It already contains `cards`, so nothing
+        # here needs the phase.
         print(json.dumps(result, indent=2, ensure_ascii=False))
         return EXIT_OK
 
+    # A phase lookup that fails must not lose the GPU numbers this command
+    # exists to print, so it falls back to the running layout. The likely cause
+    # is the job being deleted between the two calls, and the running layout is
+    # the behaviour this command had until today rather than a wrong number.
+    try:
+        done = (job or client.status(args.job_id)).get("phase") in TERMINAL_PHASES
+    except ServerError:
+        done = False
+
     progress = result.get("progress")
     if progress:
         print(f"  training       {bar(progress['percent'])}  "
@@ -1073,20 +1230,102 @@ def cmd_watch(args: argparse.Namespace) -> int:
 
     gpu = result.get("latest_gpu")
     if gpu:
-        print(f"  GPU util       {bar(gpu['utilization_percent'])}  "
-              f"{gpu['utilization_percent']}%")
-        print(f"  GPU memory     {bar(gpu['memory_percent'])}  "
-              f"{gpu['memory_used_mib']:,} / {gpu['memory_total_mib']:,} MiB "
-              f"({gpu['memory_percent']}%)")
-        print(f"  temp, power    {gpu['temperature_c']} C, {gpu['power_w']} W")
-        print(f"  samples        {len(result.get('gpu_series', []))}, "
-              f"last {result['window_seconds']}s")
+        # The reading with the most MEMORY in use. A server too old to send it
+        # leaves the last reading as the only one there is, which is what the
+        # screen falls back to as well.
+        peak = result.get("peak_gpu") or gpu
+        if done:
+            # A FINISHED RUN LEADS WITH THE PEAK. Memory is what kills a run, so
+            # the peak is the number a post-mortem came here for; the last
+            # reading is kept, because it is still evidence, but it is named for
+            # what it is instead of being labelled "GPU util". The screen was
+            # changed to this shape on 2026-09-07 after the 0%, 0 MiB readout
+            # was reported as the panel being broken.
+            print(f"  peak memory    {bar(peak['memory_percent'])}  "
+                  f"{peak['memory_used_mib']:,} / {peak['memory_total_mib']:,} MiB "
+                  f"({peak['memory_percent']}%)")
+            # ★ `peak_utilization_percent`, NOT `peak['utilization_percent']`,
+            # and the difference is the whole defect. `peak` is the sample with
+            # the most memory in it and its utilisation is whatever the card
+            # happened to be doing at that instant. On job-66b46719b854 all four
+            # A100s reached 77,631 MiB and the utilisation inside those four
+            # samples read 0, 3, 1 and 95 -- while every one of those cards
+            # actually peaked at 99 or 100 and averaged between 37.8 and 84.6.
+            # Reported on the screen 2026-09-11; the CLI never printed either
+            # figure, so it is being added correct rather than fixed.
+            print(f"  peak util      {_percent(result.get('peak_utilization_percent'))}")
+            print(f"  avg util       {_percent(result.get('avg_utilization_percent'))}")
+            print(f"  last reading   {gpu['utilization_percent']}%, "
+                  f"{gpu['memory_used_mib']:,} MiB  (run ended)")
+        else:
+            print(f"  GPU util       {bar(gpu['utilization_percent'])}  "
+                  f"{gpu['utilization_percent']}%")
+            print(f"  GPU memory     {bar(gpu['memory_percent'])}  "
+                  f"{gpu['memory_used_mib']:,} / {gpu['memory_total_mib']:,} MiB "
+                  f"({gpu['memory_percent']}%)")
+            print(f"  temp, power    {gpu['temperature_c']} C, {gpu['power_w']} W")
+        # `sample_count`, not len(gpu_series): the server thins the series to at
+        # most 400 points so a chart can draw it, and job-66b46719b854 printed
+        # 785 readings per card while this line said 393. The screen had the same
+        # defect and was fixed on 2026-09-12. A server too old to send the count
+        # falls back to the thinned length, which is the only number it has.
+        taken = result.get("sample_count") or len(result.get("gpu_series", []))
+        print(f"  samples        {taken}, last {result['window_seconds']}s")
+        print_card_table(result.get("cards") or [])
 
     if result.get("note"):
         print(f"  note           {result['note']}")
     return EXIT_OK
 
 
+def _percent(value: float | int | None) -> str:
+    """One utilisation figure, or a dash when the server did not send it.
+
+    The two utilisation fields arrived on 2026-09-10 and a server older than
+    that answers with them absent. A blank line reads as 0, which is the very
+    thing the peak figures exist to stop being claimed.
+    """
+    return "-" if value is None else f"{value}%"
+
+
+def print_card_table(cards: list[dict[str, Any]]) -> None:
+    """One row per GPU card, printed only when the job rented more than one.
+
+    WHY THIS EXISTS. Everything above describes card 0 -- `latest_gpu` and
+    `peak_gpu` are the lowest-indexed card's readings, unchanged since before
+    `cards` existed. job-66b46719b854 is four A100s in one pod, and reading its
+    CLI output left three of them invisible.
+
+    WHY ONLY ABOVE ONE CARD. With a single card these four numbers repeat the
+    block above it word for word, so the table is drawn on the same condition
+    the screen draws its own (`cards.length > 1` in ui/app.js).
+
+    Args:
+        cards: the response's `cards`, each a CardMetricsView -- `gpu_index`,
+            `series`, `latest`, `peak`, `avg_utilization_percent`,
+            `peak_utilization_percent`, `sample_count`.
+    """
+    if len(cards) < 2:
+        return
+    print(f"  {'card':<8}{'peak memory':>27}{'peak util':>11}"
+          f"{'avg util':>10}{'samples':>9}")
+    for card in cards:
+        # `peak` is this card's highest-MEMORY reading; falling back to its last
+        # one matches the screen's table and keeps a row rather than dropping a
+        # card that only ever printed once.
+        sample = card.get("peak") or card.get("latest") or {}
+        memory = "-" if not sample else (
+            f"{sample['memory_used_mib']:,} / {sample['memory_total_mib']:,} MiB "
+            f"({sample['memory_percent']:.0f}%)")
+        # Same two corrections as the block above: the card's own highest
+        # utilisation rather than the utilisation inside its memory peak, and
+        # the readings it printed rather than the points left after thinning.
+        taken = card.get("sample_count") or len(card.get("series") or [])
+        print(f"  {('GPU ' + str(card['gpu_index'])):<8}{memory:>27}"
+              f"{_percent(card.get('peak_utilization_percent')):>11}"
+              f"{_percent(card.get('avg_utilization_percent')):>10}{taken:>9}")
+
+
 def cmd_stats(args: argparse.Namespace) -> int:
     """Print this caller's team figures."""
     result = client_from_config().stats()

{hyperun-0.2.0 → hyperun-0.2.2}/hyperun/client.py

@@ -193,7 +193,8 @@ class Client:
         return self._call("GET", "/v1/stats").json()
 
     def exec_in_job(
-        self, job: str, command: str, slot: int = 0, timeout_seconds: int = 20
+        self, job: str, command: str, slot: int = 0, timeout_seconds: int = 20,
+        session: bool = False, seq: int = 0,
     ) -> dict[str, Any]:
         """Run one shell line inside a running job's workload container.
 
@@ -206,6 +207,13 @@ class Client:
             command: one shell line, run as `sh -lc <command>`.
             slot: which pod of a parallel job.
             timeout_seconds: server-side wait, capped at 25 by the server.
+            session: type the line into a shell ALREADY RUNNING in the driver
+                pod, so `cd` and exported variables survive to the next call.
+                False starts a fresh `sh -lc`, which is what a script wants.
+            seq: with `session`, the output sequence this caller last saw. The
+                reply carries the next one. It exists because the caller is a
+                different process on the server's side every time and cannot
+                hold a position in the output.
         """
         return self._call(
             "POST",
@@ -214,6 +222,8 @@ class Client:
                 "command": command,
                 "slot": slot,
                 "timeout_seconds": timeout_seconds,
+                "session": session,
+                "seq": seq,
             },
             # The server may hold the request for timeout_seconds before
             # answering; the read timeout has to outlive that on purpose.

{hyperun-0.2.0 → hyperun-0.2.2}/hyperun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperun
-Version: 0.2.0
+Version: 0.2.2
 Summary: Submit a GPU job and get results back. No kubectl, no cloud account.
 Author: DDPS Lab
 License-Expression: Apache-2.0

{hyperun-0.2.0 → hyperun-0.2.2}/pyproject.toml

@@ -34,7 +34,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hyperun"
-version = "0.2.0"
+version = "0.2.2"
 description = "Submit a GPU job and get results back. No kubectl, no cloud account."
 requires-python = ">=3.9"
 dependencies = ["requests>=2.31", "PyYAML>=6.0"]

{hyperun-0.2.0 → hyperun-0.2.2}/tests/test_cli.py

@@ -38,6 +38,15 @@ class FakeClient:
         self.estimate_result = {}
         self.validate_result = {"ok": True, "findings": [], "not_checked": []}
         self.metrics_result = {"window_seconds": 3600, "gpu_series": [], "note": ""}
+        # Every window `watch` asked for, in order. The widening of a finished
+        # job's window is invisible in the printed output -- the numbers look the
+        # same whichever window found them -- so the only way to test it is to
+        # record what was asked.
+        self.metrics_windows: list[int] = []
+        # What to answer once the window is wider than the default hour. None
+        # means "the same answer whatever the window", which is what every test
+        # written before the widening existed expects.
+        self.metrics_wide_result = None
         self.stats_result = {"team": "", "members": [], "jobs": 0, "gpu_hours": 0.0,
                              "cost_usd": 0.0, "unpriced_jobs": 0, "note": ""}
         self.secrets_result = {"names": ["GITHUB_PAT", "HF_TOKEN"],
@@ -55,6 +64,9 @@ class FakeClient:
         return self.validate_result
 
     def metrics(self, job_id, window_seconds=3600):
+        self.metrics_windows.append(window_seconds)
+        if self.metrics_wide_result is not None and window_seconds > 3600:
+            return self.metrics_wide_result
         return self.metrics_result
 
     def stats(self):
@@ -100,8 +112,8 @@ class FakeClient:
     # DDPSRUN-SHELL. Absent until 2026-09-10, which is exactly why the dispatch
     # bug below survived every release: with no double for this call, no test
     # could reach cmd_shell at all.
-    def exec_in_job(self, job, line, slot=0):
-        self.execed.append((job, line, slot))
+    def exec_in_job(self, job, line, slot=0, session=False, seq=0):
+        self.execed.append((job, line, slot, session, seq))
         return self.exec_result
 
 
@@ -475,6 +487,298 @@ def test_watch_prints_progress_and_gpu(fake, capsys):
     assert "samples        120" in printed
 
 
+def gpu_sample(util, used, total=81920, index=0):
+    """One reading, for the card tests below."""
+    return {"utilization_percent": util, "memory_used_mib": used,
+            "memory_total_mib": total, "memory_percent": round(100 * used / total, 1),
+            "temperature_c": 62, "power_w": 310.0, "gpu_index": index}
+
+
+def four_a100_cards():
+    """job-66b46719b854 as the server reports it: four A100s in one pod.
+
+    The numbers are that run's: every card peaked at 77,631 of 81,920 MiB, and
+    the utilisation INSIDE those four highest-memory samples read 0, 3, 1 and 95
+    while the cards themselves peaked at 100, 99, 100 and 99. That gap is the
+    defect the screen carried until 2026-09-11, so the fixture keeps it.
+    """
+    peaks = [(0, 100, 84.6), (3, 99, 37.8), (1, 100, 79.2), (95, 99, 81.3)]
+    return [
+        {"gpu_index": i,
+         "series": [{}] * 393,
+         "latest": gpu_sample(0, 0, index=i),
+         "peak": gpu_sample(at_peak, 77631, index=i),
+         "avg_utilization_percent": avg,
+         "peak_utilization_percent": real_peak,
+         "sample_count": 785}
+        for i, (at_peak, real_peak, avg) in enumerate(peaks)
+    ]
+
+
+def test_every_card_is_printed_not_only_the_first(fake, capsys):
+    # ★ THE DEFECT. `latest_gpu` is card 0, so a four-A100 job printed one card
+    # and three quarters of a $44 run was invisible from the CLI. The screen was
+    # fixed on 2026-09-08 and this surface was not.
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200),
+        "peak_gpu": gpu_sample(0, 77631),
+        "gpu_series": [{}] * 393,
+        "sample_count": 785,
+        "peak_utilization_percent": 100,
+        "avg_utilization_percent": 84.6,
+        "cards": four_a100_cards(),
+        "window_seconds": 604800, "note": "",
+    }
+    assert run(["watch", "job-66b46719b854"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    # `GPU 0` is a table row; `GPU util` and `GPU memory` are the headline above
+    # it, so the card index is what tells the two apart.
+    rows = {line.split()[1]: line.split()
+            for line in printed.splitlines()
+            if line.startswith("  GPU ") and line.split()[1].isdigit()}
+    assert sorted(rows) == ["0", "1", "2", "3"]
+
+    # A row reads: GPU <n> <used> / <total> MiB (<pct>%) <peak util> <avg> <n>
+    # so the last three fields are the three that were wrong or missing.
+    expected = {"0": ("100%", "84.6%"), "1": ("99%", "37.8%"),
+                "2": ("100%", "79.2%"), "3": ("99%", "81.3%")}
+    for index, (peak_util, avg_util) in expected.items():
+        fields = rows[index]
+        # Each card's own peak memory, not card 0's repeated four times.
+        assert fields[2:7] == ["77,631", "/", "81,920", "MiB", "(95%)"]
+        # ★ `peak_utilization_percent`, NOT the utilisation inside the memory
+        # peak. The two differ on this very run: the four highest-memory samples
+        # read 0%, 3%, 1% and 95% while the cards peaked at 100, 99, 100 and 99.
+        assert fields[7] == peak_util
+        assert fields[8] == avg_util
+        # The readings this card printed, not the 393 points left after the
+        # server thinned the series for the chart.
+        assert fields[9] == "785"
+
+
+def test_one_card_prints_no_table_because_it_would_repeat_the_block(fake, capsys):
+    # With a single card the table's four numbers are the same four printed
+    # directly above it, so it is drawn on the same condition the screen uses.
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "gpu_series": [{}] * 120,
+        "cards": [{"gpu_index": 0, "series": [{}] * 120,
+                   "latest": gpu_sample(94, 38200, total=45440),
+                   "peak": gpu_sample(94, 38200, total=45440),
+                   "avg_utilization_percent": 90.0,
+                   "peak_utilization_percent": 99,
+                   "sample_count": 120}],
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "peak memory" not in printed      # the table's header row
+    assert "38,200 / 45,440 MiB" in printed
+
+
+def test_a_finished_job_leads_with_the_peak_not_the_idle_last_reading(fake, capsys):
+    # ★ THE DEFECT. A finished job's last reading is the idle card in the
+    # seconds before teardown -- 0%, 0 MiB -- and printing it as "GPU util" says
+    # the run was idle. What a post-mortem asks for is the peak, because running
+    # out of memory is what kills a run. The screen was changed to this shape on
+    # 2026-09-07.
+    fake.status_result = {"job_id": "job-66b46719b854", "name": "baseline-c",
+                          "phase": "Succeeded"}
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(0, 0),
+        "peak_gpu": gpu_sample(0, 77631),
+        "gpu_series": [{}] * 393,
+        "sample_count": 785,
+        "peak_utilization_percent": 100,
+        "avg_utilization_percent": 84.6,
+        "window_seconds": 604800, "note": "",
+    }
+    assert run(["watch", "job-66b46719b854"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "peak memory" in printed
+    assert "77,631 / 81,920 MiB" in printed
+    # The peak of the CARD, not the utilisation inside the highest-memory
+    # sample, which on this run was 0 while the card reached 100.
+    assert "peak util      100%" in printed
+    assert "avg util       84.6%" in printed
+    # The last reading is still shown, and named for what it is.
+    assert "last reading   0%, 0 MiB  (run ended)" in printed
+    # The running job's labels are gone: leaving "GPU util  0%" on the screen is
+    # exactly the claim this fix removes.
+    assert "GPU util" not in printed
+
+
+def test_a_running_job_keeps_the_live_readings_as_the_headline(fake, capsys):
+    # The mirror of the test above. `peak_gpu` being present is not on its own a
+    # finished job -- the server sends it for a running one too -- so the phase
+    # is what decides, and a Running job still leads with what the card is doing
+    # NOW.
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "peak_gpu": gpu_sample(99, 40000, total=45440),
+        "gpu_series": [{}] * 120,
+        "peak_utilization_percent": 99,
+        "avg_utilization_percent": 90.0,
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "GPU util" in printed
+    assert "38,200 / 45,440 MiB" in printed
+    assert "last reading" not in printed
+
+
+def test_a_phase_lookup_that_fails_still_prints_the_gpu_numbers(fake, capsys):
+    # The numbers are what the command exists for, so a job deleted between the
+    # metrics call and the phase call falls back to the running layout rather
+    # than exiting with nothing printed.
+    def refuse(job_id):
+        raise ServerError("404: no such job")
+
+    fake.status = refuse
+    fake.metrics_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "gpu_series": [{}] * 120,
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert "38,200 / 45,440 MiB" in capsys.readouterr().out
+
+
+def _hours_ago(hours):
+    """An RFC 3339 timestamp that many hours in the past, as the server writes it."""
+    import datetime as _dt
+    t = _dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(hours=hours)
+    return t.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def test_watch_widens_the_window_to_reach_a_finished_jobs_readings(fake, capsys):
+    # ★ THE DEFECT: nothing is stored, so `watch` re-reads the log and the window
+    # is measured back from NOW. A run that ended five hours ago has every
+    # reading outside the default hour, and `watch` printed the phase and no
+    # numbers -- the phase being the one thing the reader already knew.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "name": "bank-exp2",
+                          "phase": "Succeeded", "started_at": _hours_ago(9),
+                          "finished_at": _hours_ago(5)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+    fake.metrics_wide_result = {
+        "latest_gpu": gpu_sample(94, 38200, total=45440),
+        "peak_gpu": gpu_sample(94, 38200, total=45440),
+        "peak_utilization_percent": 94,
+        "gpu_series": [{}] * 120, "window_seconds": 36000, "note": "",
+    }
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert len(fake.metrics_windows) == 2, "the empty first window was not retried"
+    assert fake.metrics_windows[0] == 3600
+    # Nine hours of run plus a minute of headroom, so the FIRST reading is inside
+    # the window rather than exactly on its edge.
+    assert fake.metrics_windows[1] >= 9 * 3600
+    assert "38,200 / 45,440 MiB" in capsys.readouterr().out
+
+
+def test_a_window_the_caller_chose_is_never_widened(fake):
+    # An explicit --window is an instruction, not a default. Widening it would
+    # answer a question the caller did not ask, and the answer would be a bigger
+    # read of the log than they wanted.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Succeeded",
+                          "started_at": _hours_ago(9)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07", "--window", "600"]) == cli.EXIT_OK
+    assert fake.metrics_windows == [600]
+
+
+def test_a_running_job_is_not_widened_however_quiet_it_is(fake):
+    # A running job's readings ARE inside the last hour, so an empty answer means
+    # the job has printed nothing yet. Re-reading a week of its log would buy a
+    # second empty answer and a much larger response.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Running",
+                          "started_at": _hours_ago(9)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert fake.metrics_windows == [3600]
+
+
+def test_the_widened_window_stops_at_the_servers_own_cap(fake):
+    # `window_seconds` is declared ge=60, le=604800 on the metrics route. Asking
+    # for more is a 422, so a job that started a month ago must be clamped here
+    # rather than turned into an error the reader cannot act on.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Failed",
+                          "started_at": _hours_ago(24 * 30)}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert fake.metrics_windows[1] == cli.MAX_WATCH_WINDOW == 604800
+
+
+def test_a_finished_job_with_no_start_time_leaves_the_window_alone(fake):
+    # Nothing to compute a width from. Widening to the cap "just in case" would
+    # read seven days of log off the back of a missing field.
+    fake.status_result = {"job_id": "job-a8acdef80a07", "phase": "Succeeded"}
+    fake.metrics_result = {"gpu_series": [], "window_seconds": 3600, "note": ""}
+
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert fake.metrics_windows == [3600]
+
+
+def test_the_window_help_quotes_the_servers_real_cap(capsys):
+    # It said 86400 and the server accepts 604800, so the help talked a reader
+    # out of a window the server would have answered.
+    with pytest.raises(SystemExit):
+        cli.main(["watch", "--help"])
+    printed = capsys.readouterr().out
+    assert "604800" in printed
+    assert "86400" not in printed
+
+
+def test_watch_json_prints_the_cards_and_asks_for_no_phase(fake, capsys):
+    # --json is the server's answer verbatim, so the second request would buy
+    # nothing. It is one GET per typed `watch` and worth not making.
+    def refuse(job_id):
+        raise AssertionError("--json must not ask for the phase")
+
+    fake.status = refuse
+    fake.metrics_result = {"cards": four_a100_cards(), "gpu_series": [],
+                           "window_seconds": 3600, "note": ""}
+    assert run(["watch", "job-66b46719b854", "--json"]) == cli.EXIT_OK
+    assert len(json.loads(capsys.readouterr().out)["cards"]) == 4
+
+
+def test_samples_counts_the_readings_taken_not_the_chart_points(fake, capsys):
+    # ★ THE SAME DEFECT THE SCREEN HAD. The server thins the series to at most
+    # 400 points so a chart can draw it; job-66b46719b854 printed 785 readings
+    # per card and both surfaces counted the 393 that survived. The mean and the
+    # peak are computed over all 785, so the count beside them has to be 785.
+    fake.metrics_result = {
+        "latest_gpu": {"utilization_percent": 94, "memory_used_mib": 38200,
+                       "memory_total_mib": 45440, "memory_percent": 84.1,
+                       "temperature_c": 71, "power_w": 298.5},
+        "gpu_series": [{}] * 393,
+        "sample_count": 785,
+        "window_seconds": 604800, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    printed = capsys.readouterr().out
+    assert "samples        785" in printed
+    assert "samples        393" not in printed
+
+
+def test_a_server_too_old_to_send_the_count_still_prints_one(fake, capsys):
+    # The field arrived on 2026-09-12. Against a server that predates it the
+    # thinned length is the only number there is, and a blank is worse.
+    fake.metrics_result = {
+        "latest_gpu": {"utilization_percent": 10, "memory_used_mib": 1,
+                       "memory_total_mib": 2, "memory_percent": 50.0,
+                       "temperature_c": 40, "power_w": 50.0},
+        "gpu_series": [{}] * 12,
+        "window_seconds": 3600, "note": "",
+    }
+    assert run(["watch", "job-a8acdef80a07"]) == cli.EXIT_OK
+    assert "samples        12" in capsys.readouterr().out
+
+
 def test_an_unsettled_projection_is_labelled_rather_than_stated(fake, capsys):
     fake.metrics_result = {
         "progress": {"step": 5, "total_steps": 556, "percent": 0.9,
@@ -820,7 +1124,8 @@ def test_the_subcommand_name_survives_the_shell_positional():
 
 def test_shell_with_a_command_runs_it_once_and_returns_its_exit_code(fake, capsys):
     assert run(["shell", "job-a8acdef80a07", "--", "date"]) == 0
-    assert fake.execed == [("job-a8acdef80a07", "date", 0)]
+    assert fake.execed == [("job-a8acdef80a07", "date", 0, False, 0)], (
+        "`-- command` 형태는 stateless 다 — script 는 남길 session 이 없다")
     assert "Sep 10" in capsys.readouterr().out
 
 
@@ -858,3 +1163,62 @@ def test_every_other_subcommand_still_dispatches(fake):
     """dest 를 바꾼 것이 나머지를 깨지 않았다는 확인."""
     assert run(["status", "job-a8acdef80a07"]) == 0
     assert run(["secrets"]) == 0
+
+
+def test_the_prompt_uses_a_session_and_carries_the_sequence(fake, monkeypatch, capsys):
+    """★ WHY THE PROMPT DIFFERS FROM `-- command`. A person at a prompt assumes
+    `cd` sticks; a script does not want a session left behind for the idle timer.
+    So the prompt sends session=True and threads the sequence through, and the
+    one-shot form stays stateless."""
+    typed = iter(["cd /workspace", "pwd", "exit"])
+    monkeypatch.setattr("builtins.input", lambda _prompt: next(typed))
+    fake.exec_result = {"output": "/workspace\n", "exit_code": 0, "seq": 11, "lost": False}
+
+    assert run(["shell", "job-a8acdef80a07"]) == 0
+    assert [(line, sess) for _job, line, _slot, sess, _seq in fake.execed] == [
+        ("cd /workspace", True), ("pwd", True)]
+    # The FIRST line starts at 0 and the second carries what the first returned.
+    assert [seq for *_rest, seq in fake.execed] == [0, 11]
+
+
+def test_a_session_that_closed_is_reopened_once_and_the_person_is_told(fake, monkeypatch,
+                                                                       capsys):
+    """The one case where starting a new shell is right: the old one is provably
+    gone (ten minutes unread, or the workload restarted). Everywhere else a new
+    shell silently loses the `cd`, which is the failure this feature exists to end
+    -- so the message says what was lost."""
+    typed = iter(["pwd", "exit"])
+    monkeypatch.setattr("builtins.input", lambda _prompt: next(typed))
+
+    calls = {"n": 0}
+    original = fake.exec_in_job
+
+    def flaky(job, line, slot=0, session=False, seq=0):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise cli.ServerError("no session for this slot; open one first")
+        return original(job, line, slot=slot, session=session, seq=seq)
+
+    fake.exec_in_job = flaky
+    assert run(["shell", "job-a8acdef80a07"]) == 0
+    assert calls["n"] == 2, "it retried once rather than giving up or looping"
+    err = capsys.readouterr().err
+    assert "reopening" in err and "are gone" in err, (
+        "a silent reopen would let somebody keep typing paths relative to a cd that no "
+        "longer applies")
+
+
+def test_a_failure_that_is_not_a_lost_session_is_not_retried(fake, monkeypatch, capsys):
+    typed = iter(["pwd", "exit"])
+    monkeypatch.setattr("builtins.input", lambda _prompt: next(typed))
+
+    calls = {"n": 0}
+
+    def always_bad(job, line, slot=0, session=False, seq=0):
+        calls["n"] += 1
+        raise cli.ServerError("the driver pod did not answer in JSON")
+
+    fake.exec_in_job = always_bad
+    assert run(["shell", "job-a8acdef80a07"]) == 0
+    assert calls["n"] == 1, "retrying an unrelated failure would double every bad command"
+    assert "did not answer in JSON" in capsys.readouterr().err

{hyperun-0.2.0 → hyperun-0.2.2}/tests/test_client.py

@@ -65,7 +65,9 @@ def test_shell_sends_the_command_and_waits_longer_than_the_server():
     result = client_with(session).exec_in_job("baseline-c", "nvidia-smi -L", slot=1)
     call = session.calls[0]
     assert call["url"] == "https://run.example/v1/jobs/baseline-c/exec"
-    assert call["json"] == {"command": "nvidia-smi -L", "slot": 1, "timeout_seconds": 20}
+    assert call["json"] == {"command": "nvidia-smi -L", "slot": 1, "timeout_seconds": 20,
+                            "session": False, "seq": 0}, (
+        "PACSRUN-SHELL-SESSION 이후에도 기본은 한 줄짜리 stateless 형태다")
     # The server may hold the request for its whole window, so the client's
     # read timeout must outlive it.
     assert call["timeout"] > 20
@@ -155,3 +157,15 @@ def test_a_submit_is_given_more_patience_than_a_read():
     read_session = FakeSession(FakeResponse(200, {}))
     client_with(read_session).status("job-a8acdef80a07")
     assert submit_session.calls[0]["timeout"] > read_session.calls[0]["timeout"]
+
+
+def test_shell_can_ask_for_the_persistent_session_and_carries_its_sequence():
+    """The prompt loop's shape: the same route, with `session` and the output
+    sequence the caller last saw. Without the sequence the caller would be handed
+    everything the shell has ever printed on every line."""
+    session = FakeSession(FakeResponse(200, {"output": "/workspace\n", "exit_code": 0,
+                                             "seq": 42, "lost": False, "note": ""}))
+    result = client_with(session).exec_in_job("baseline-c", "pwd", session=True, seq=7)
+    assert session.calls[0]["json"]["session"] is True
+    assert session.calls[0]["json"]["seq"] == 7
+    assert result["seq"] == 42, "the reply carries the next one back"