calkit-python 0.41.17__py3-none-any.whl → 0.41.18__py3-none-any.whl

calkit/cli/scheduler.py

@@ -304,6 +304,25 @@ def _slurm_exit_code(job_id: str) -> int | None:
     return None
 
 
+def _parse_pbs_state(stdout: str) -> tuple[str | None, int | None]:
+    # `qstat -f` output is a flat list of `key = value` lines; pull out the
+    # job's state and, for a finished job, its exit status. The status field is
+    # spelled `Exit_status` on PBS Pro and `exit_status` on some Torque builds,
+    # so match it case-insensitively.
+    state = None
+    exit_code = None
+    for line in stdout.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("job_state"):
+            state = stripped.split("=", 1)[-1].strip()
+        elif stripped.lower().startswith("exit_status"):
+            try:
+                exit_code = int(stripped.split("=", 1)[-1].strip())
+            except ValueError:
+                exit_code = None
+    return state, exit_code
+
+
 def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
     """Return ``(active, exit_code)`` for a scheduler job.
 
@@ -339,39 +358,51 @@ def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
         if "invalid job id" in stderr:
             return False, _slurm_exit_code(job_id)
         return True, None
-    # Use `qstat -f` and parse job_state: on Torque/OpenPBS, plain `qstat
-    # <id>` returns exit 0 even for completed (C) jobs, so checking the
-    # return code alone would cause `calkit sched batch` to hang forever
-    # after a PBS job finishes.  States C and F mean the job is done, and a
-    # finished job's record carries its `Exit_status`.
+    # PBS reports a finished job differently across variants, so poll in two
+    # steps that mirror the SLURM path above. First consult the active-queue
+    # view with `qstat -f`: every live state (R, Q, H, E) shows here, and on
+    # Torque a completed job lingers here in state `C` carrying its
+    # `Exit_status`. `qstat -f` is used rather than plain `qstat <id>` because
+    # the latter exits 0 even for completed jobs, so the return code alone
+    # could not tell running from done.
     p = subprocess.run(
         ["qstat", "-f", job_id], capture_output=True, text=True, check=False
     )
-    if p.returncode != 0:
-        # A non-zero exit is ambiguous: either the job is gone (completed and
-        # purged from history) or `qstat` itself failed transiently---a busy
-        # PBS server periodically refuses connections or times out. Treating a
-        # transient failure as completion would stop the wait while the job is
-        # still running and writing its log, so only conclude the job is done
-        # when qstat positively reports it is unknown; otherwise keep waiting.
-        # A purged job (unknown to qstat) is done but with an unknowable exit
-        # status; any other error is transient, so report the job as active.
-        stderr = (p.stderr or "").lower()
-        return ("unknown job" not in stderr), None
-    state = None
-    exit_code = None
-    for line in p.stdout.splitlines():
-        stripped = line.strip()
-        if stripped.startswith("job_state"):
-            state = stripped.split("=", 1)[-1].strip()
-        elif stripped.lower().startswith("exit_status"):
-            try:
-                exit_code = int(stripped.split("=", 1)[-1].strip())
-            except ValueError:
-                exit_code = None
-    if state in ("C", "F"):
-        return False, exit_code
-    return True, None
+    if p.returncode == 0:
+        state, exit_code = _parse_pbs_state(p.stdout)
+        if state in ("C", "F"):
+            return False, exit_code
+        if state is not None:
+            return True, None
+    # The job is no longer in the active queue---on PBS Pro a finished job is
+    # hidden from plain `qstat` and shows (as state `F` with its `Exit_status`)
+    # only under `qstat -x`---or qstat failed. Ask the history view for the
+    # finished record. Without this, a PBS Pro job that has finished is never
+    # seen as done and the wait hangs forever. `-x` means "include finished
+    # jobs" on PBS Pro; on Torque it switches qstat to XML output, which simply
+    # won't parse here, leaving the stderr check below to settle it (Torque
+    # already reported completion above via state `C`).
+    hist = subprocess.run(
+        ["qstat", "-x", "-f", job_id],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if hist.returncode == 0:
+        state, exit_code = _parse_pbs_state(hist.stdout)
+        if state in ("C", "F"):
+            return False, exit_code
+        if state is not None:
+            return True, None
+    # Neither view resolved a state. A non-zero exit is ambiguous: either the
+    # job is gone (finished and purged from history) or qstat failed
+    # transiently---a busy PBS server periodically refuses connections or times
+    # out. Treating a transient failure as completion would stop the wait while
+    # the job is still running and writing its log, so only conclude the job is
+    # done when qstat positively reports it is unknown; otherwise keep waiting.
+    # A purged job is done but with an unknowable exit status.
+    stderr = (p.stderr or "").lower()
+    return ("unknown job" not in stderr), None
 
 
 def _is_active(kind: str, job_id: str) -> bool:

calkit/tests/cli/test_scheduler.py

@@ -215,44 +215,75 @@ def test_poll_job_pbs(monkeypatch):
 
     import calkit.cli.scheduler as sched
 
-    outcomes: dict = {}
+    # PBS is polled in two steps: plain `qstat -f` for the active queue, then
+    # `qstat -x -f` for the finished-job history view. The fake dispatches on
+    # whether `-x` is present so each step can be simulated independently, the
+    # way a real PBS Pro server answers them.
+    active: dict = {}
+    history: dict = {}
 
-    def _fake_run(*args, **kwargs):
+    def _fake_run(cmd, *args, **kwargs):
+        outcomes = history if "-x" in cmd else active
         return subprocess.CompletedProcess(
-            args=args[0],
+            cmd,
             returncode=outcomes["returncode"],
             stdout=outcomes.get("stdout", ""),
             stderr=outcomes.get("stderr", ""),
         )
 
     monkeypatch.setattr(sched.subprocess, "run", _fake_run)
-    # A running job (state R) is active with no exit code yet.
-    outcomes.update(returncode=0, stdout="    job_state = R\n", stderr="")
+    # A running job (state R) is active with no exit code yet, and resolves
+    # from the active queue alone---the history view is never consulted.
+    active.update(returncode=0, stdout="    job_state = R\n", stderr="")
+    history.clear()
     assert _poll_job("pbs", "1.pbs") == (True, None)
-    # Terminal states C (Torque) and F (PBS Pro) mean the job is done; qstat
-    # still exits 0 while the record lingers in history and carries the exit
-    # status, which is reported back so the caller can fail a bad job.
-    outcomes.update(
+    # The reported bug: on PBS Pro a finished job leaves the active queue and
+    # is shown only under `qstat -x`, where it appears as state F with its
+    # Exit_status. Plain `qstat -f` errors with a message that is NOT "unknown
+    # job", which used to be read as "still active" and hang the wait forever.
+    active.update(
+        returncode=1,
+        stdout="",
+        stderr="qstat: 1.pbs Job has finished, use -x to obtain historical "
+        "job information\n",
+    )
+    history.update(
         returncode=0, stdout="    job_state = F\n    Exit_status = 0\n"
     )
     assert _poll_job("pbs", "1.pbs") == (False, 0)
-    outcomes.update(
+    # A non-zero exit code from the finished job is reported back so the caller
+    # can fail the stage.
+    history.update(
+        returncode=0, stdout="    job_state = F\n    Exit_status = 137\n"
+    )
+    assert _poll_job("pbs", "1.pbs") == (False, 137)
+    # On Torque a completed job lingers in the active queue as state C with its
+    # exit status, so it is done without ever needing the history view.
+    active.update(
         returncode=0, stdout="    job_state = C\n    exit_status = 137\n"
     )
+    history.update(returncode=1, stdout="", stderr="should not be reached\n")
     assert _poll_job("pbs", "1.pbs") == (False, 137)
     # A finished job whose record lacks an exit status yields an unknown code.
-    outcomes.update(returncode=0, stdout="    job_state = F\n")
+    active.update(returncode=1, stdout="", stderr="qstat: Unknown Job Id\n")
+    history.update(returncode=0, stdout="    job_state = F\n")
     assert _poll_job("pbs", "1.pbs") == (False, None)
-    # Once purged, qstat exits non-zero and reports the job unknown: done, but
-    # with no way to recover the exit status.
-    outcomes.update(
+    # Once purged from history too, both views error with "unknown job": done,
+    # but with no way to recover the exit status.
+    active.update(
+        returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
+    )
+    history.update(
         returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
     )
     assert _poll_job("pbs", "1.pbs") == (False, None)
-    # A transient qstat failure (busy/unreachable server) is NOT completion:
-    # treating it as done would stop the wait while the job still runs, so the
-    # job is reported active and the caller keeps polling.
-    outcomes.update(
+    # A transient qstat failure (busy/unreachable server) on both views is NOT
+    # completion: treating it as done would stop the wait while the job still
+    # runs, so the job is reported active and the caller keeps polling.
+    active.update(
+        returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
+    )
+    history.update(
         returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
     )
     assert _poll_job("pbs", "1.pbs") == (True, None)

{calkit_python-0.41.17.dist-info → calkit_python-0.41.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.17
+Version: 0.41.18
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.17.dist-info → calkit_python-0.41.18.dist-info}/RECORD

@@ -44,7 +44,7 @@ calkit/cli/new.py,sha256=OtXcXmhtywXM54dgyjxQgtYEfCYBAi_UVKvMlPjSI8k,129111
 calkit/cli/notebooks.py,sha256=mMNHE8yT0BDN0iJE07GCBFAseyDF0jSRshIm1CQjgrY,24645
 calkit/cli/office.py,sha256=jVSTvbl91TCzlglST5O2b8hdApZA_QHw_UiEQFJqxdc,1754
 calkit/cli/overleaf.py,sha256=bS-SkYsSqqhC9YsP-NCzpYJCGS3820w18w_s8JOFihg,25216
-calkit/cli/scheduler.py,sha256=Mn_G2tx78A_wJ3XYE0hRtTkPj376sKTWpSxMZKirHuA,38660
+calkit/cli/scheduler.py,sha256=Nt1wQMJzuY3ZNkq1KVEV0dZk4T8Gr0XPLQ6mJpXPrZE,40125
 calkit/cli/update.py,sha256=KaO-VbUxMpR_a9FV42HG8d9p-iaA4-2-L-fAgb2LiYI,43499
 calkit/cli/main/__init__.py,sha256=qu1POZPyqs33ZKfasOxv_Wc-EzVcEgK3Dt_vwFL8Bi8,65
 calkit/cli/main/core.py,sha256=MMKEAI4bAJm1YJcS4E5MTQAR-XluwtGn3wlsNYjK-NE,123878
@@ -103,7 +103,7 @@ calkit/tests/cli/test_list.py,sha256=ejMg6q8vtjCbKxvS9zZ32rv_oeKH94JbvpEiVjnw5QA
 calkit/tests/cli/test_new.py,sha256=_-lQH53OkKC-nyLPrdnwndTDyzXv8-aGnXOaDQ_bu-s,46568
 calkit/tests/cli/test_notebooks.py,sha256=2t1KiGhEz9H9LcIdWy4jGM05REUxJpWiIg6fSpX12ME,10678
 calkit/tests/cli/test_overleaf.py,sha256=Y-ud4kZHh3nhyoZlXrtMfx61z-GPNytUpfDHMad3tuA,16474
-calkit/tests/cli/test_scheduler.py,sha256=MY2tmhcDLPypNS2yFONUeLjvQog37iw2XGHc9c2wtCY,15715
+calkit/tests/cli/test_scheduler.py,sha256=UBJCcZzx9lvPwymFi3SOt5bdwuNZ8wzDy2obMZ-Qn5E,17205
 calkit/tests/cli/test_update.py,sha256=bwRF0kpqbCVxNvGH777LHFX3oUP2DyN3XXUBDCEKNoc,6356
 calkit/tests/cli/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/tests/cli/main/test_core.py,sha256=lowrbL1WCUPAEQjzMWFLPTbrARyFGEQKtVWGp2Lqrpg,63418
@@ -123,23 +123,23 @@ calkit/tests/models/test_pipeline.py,sha256=AJsMAX4928U3_Omt0_8K96U5LU_KoNErgpHe
 calkit/agent_skills/add-pipeline-stage/SKILL.md,sha256=kpYYb-rJsS4B9p1Ub3gL21irCG5ZZ4DvWZpdJeEwPZk,3772
 calkit/agent_skills/conventions/SKILL.md,sha256=nAM2FjSMk6ED56Dr5zh2bL_dhfzDra-h2ZgzHU7ruS4,9524
 calkit/agent_skills/create-pipeline/SKILL.md,sha256=2FGw1iDQVRk5FUq79GP3lPdgvgof2Wi1O9O-abGGtgs,5422
-calkit_python-0.41.17.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/package.json,sha256=39PQEWOzw0qtqD7Kep3OPbUR113vI7CG5TeD84v7clM,6224
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js,sha256=rJA1dk1bKtu1QkURrp5HEC2WCxzZppPV1SCYz-35GTc,8737
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
-calkit_python-0.41.17.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
-calkit_python-0.41.17.dist-info/METADATA,sha256=wqrw0yWOq-fJqDiURfAMqy0zMP4MSuBHuZAMlsC3ddo,12437
-calkit_python-0.41.17.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-calkit_python-0.41.17.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
-calkit_python-0.41.17.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
-calkit_python-0.41.17.dist-info/RECORD,,
+calkit_python-0.41.18.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/package.json,sha256=39PQEWOzw0qtqD7Kep3OPbUR113vI7CG5TeD84v7clM,6224
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js,sha256=rJA1dk1bKtu1QkURrp5HEC2WCxzZppPV1SCYz-35GTc,8737
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
+calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
+calkit_python-0.41.18.dist-info/METADATA,sha256=zexxBw6ZYpD61wV5bcv8i25G0lGpGxrhJ0HKM3CAh1s,12437
+calkit_python-0.41.18.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+calkit_python-0.41.18.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
+calkit_python-0.41.18.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
+calkit_python-0.41.18.dist-info/RECORD,,