calkit-python 0.41.12__py3-none-any.whl → 0.41.14__py3-none-any.whl

calkit/cli/delete.py

@@ -0,0 +1,33 @@
+"""CLI for deleting objects."""
+
+from __future__ import annotations
+
+import typer
+
+import calkit
+from calkit.cli import AliasGroup, raise_error
+from calkit.core import ryaml
+
+delete_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
+
+
+@delete_app.command(name="question")
+def delete_question(
+    index: int = typer.Argument(
+        ...,
+        help="1-based index of the question to remove (see `calkit list questions`).",
+    ),
+):
+    """Remove a question by its 1-based index."""
+    ck_info = calkit.load_calkit_info()
+    questions = ck_info.get("questions", []) or []
+    if index < 1 or index > len(questions):
+        raise_error(
+            f"Invalid question index {index}; "
+            f"there are {len(questions)} question(s)."
+        )
+    removed = questions.pop(index - 1)
+    ck_info["questions"] = questions
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    typer.echo(f"Removed question: {removed}")

calkit/cli/list.py

@@ -55,7 +55,7 @@ def _list_objects(
 
 
 def _list_artifacts(
-    kind: Literal["figures", "datasets"],
+    kind: Literal["figures", "datasets", "results", "presentations"],
     json_output: bool,
     declared_only: bool,
 ):
@@ -134,6 +134,61 @@ def list_datasets(
     _list_artifacts("datasets", json_output, declared_only)
 
 
+@list_app.command(name="results")
+def list_results(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list results declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
+    """List results in the project."""
+    _list_artifacts("results", json_output, declared_only)
+
+
+@list_app.command(name="presentations|pres")
+def list_presentations(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list presentations declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
+    """List presentations in the project."""
+    _list_artifacts("presentations", json_output, declared_only)
+
+
+@list_app.command(name="questions")
+def list_questions(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+):
+    """List the project's questions (1-indexed)."""
+    questions = calkit.load_calkit_info().get("questions", []) or []
+    if json_output:
+        typer.echo(json.dumps(questions))
+        return
+    for n, question in enumerate(questions, start=1):
+        typer.echo(f"{n}. {question}")
+
+
 @list_app.command(name="publications|pubs")
 def list_publications():
     """List publications in the project."""

calkit/cli/main/core.py

@@ -48,6 +48,7 @@ from calkit.cli.check import (
 )
 from calkit.cli.cloud import cloud_app
 from calkit.cli.config import config_app
+from calkit.cli.delete import delete_app
 from calkit.cli.describe import describe_app
 from calkit.cli.dev import dev_app
 from calkit.cli.import_ import import_app
@@ -69,6 +70,7 @@ app = typer.Typer(
 )
 app.add_typer(config_app, name="config", help="Configure Calkit.")
 app.add_typer(new_app, name="new|create", help="Create a new Calkit object.")
+app.add_typer(delete_app, name="delete|rm", help="Delete a Calkit object.")
 app.add_typer(
     notebooks_app, name="notebooks|nb", help="Work with Jupyter notebooks."
 )
@@ -1172,7 +1174,10 @@ def pull(
         and "-R" not in dvc_args
     ):
         dvc_args.append("--recursive")
-    result = calkit.dvc.run_dvc_command(["pull"] + dvc_args)
+    result = calkit.dvc.run_dvc_command(
+        ["pull"] + dvc_args,
+        lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
+    )
     if result != 0:
         raise_error("DVC pull failed")
     calkit.dvc.zip.sync_all(direction="to-workspace")
@@ -1187,7 +1192,9 @@ def pull(
                 continue
             typer.echo(f"DVC pulling subproject: {sp_path}")
             sp_result = calkit.dvc.run_dvc_command(
-                ["pull"] + dvc_args, cwd=sp_path
+                ["pull"] + dvc_args,
+                cwd=sp_path,
+                lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
             )
             if sp_result != 0:
                 raise_error(f"DVC pull failed for subproject: {sp_path}")
@@ -1224,7 +1231,10 @@ def push(
                     calkit.dvc.set_remote_auth(remote_name=name)
         if remotes:
             typer.echo("Pushing to DVC remote")
-            result = calkit.dvc.run_dvc_command(["push"] + dvc_args)
+            result = calkit.dvc.run_dvc_command(
+                ["push"] + dvc_args,
+                lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
+            )
             if result != 0:
                 raise_error("DVC push failed")
         else:
@@ -2033,7 +2043,10 @@ def run(
             original_dir = os.getcwd()
             try:
                 os.chdir(sp_path)
-                sp_res = dvc_cli_main(["repro"] + sp_args)
+                with calkit.dvc.dvc_lock_timeout(
+                    calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT
+                ):
+                    sp_res = dvc_cli_main(["repro"] + sp_args)
             finally:
                 os.chdir(original_dir)
             if sp_res != 0:
@@ -2102,7 +2115,15 @@ def run(
     if force:
         os.environ["CALKIT_FORCE"] = "1"
     try:
-        res = dvc_cli_main(["repro"] + args)
+        # Wait generously for the repo lock instead of failing after DVC's 3s
+        # default. Brief contention is common and benign: a background
+        # `calkit status` poller (e.g. the VS Code extension), a stage whose
+        # command is itself `calkit run`, or DVC's own per-stage re-lock (DVC
+        # releases the repo lock while a stage command runs and re-acquires it
+        # the instant it finishes) can all hold it momentarily. Without this,
+        # such a collision aborts the whole run with "Unable to acquire lock".
+        with calkit.dvc.dvc_lock_timeout(calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT):
+            res = dvc_cli_main(["repro"] + args)
     finally:
         os.environ.pop("CALKIT_FORCE", None)
     failed = failed or res != 0

calkit/cli/new.py

@@ -9,6 +9,7 @@ import shutil
 import subprocess
 import time
 from enum import Enum
+from typing import Literal
 
 import typer
 from typing_extensions import Annotated
@@ -591,6 +592,102 @@ def new_figure(
             repo.git.commit(["-m", f"Add figure {path}"])
 
 
+def _new_simple_artifact(
+    kind: Literal["results", "presentations"],
+    path: str,
+    title: str,
+    description: str | None,
+    stage_name: str | None,
+    no_commit: bool,
+    overwrite: bool,
+) -> None:
+    """Declare a simple artifact (path/title/description/stage) in calkit.yaml."""
+    singular = kind.rstrip("s")
+    ck_info = calkit.load_calkit_info()
+    objects = ck_info.get(kind, [])
+    paths = [o.get("path") for o in objects]
+    if not overwrite and path in paths:
+        raise_error(f"{singular.capitalize()} at path {path} already exists")
+    if overwrite and path in paths:
+        objects = [o for o in objects if o.get("path") != path]
+    obj = dict(path=path, title=title)
+    if description is not None:
+        obj["description"] = description
+    if stage_name is not None:
+        obj["stage"] = stage_name
+    objects.append(obj)
+    ck_info[kind] = objects
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if not no_commit:
+        repo = calkit.git.get_repo()
+        repo.git.add("calkit.yaml")
+        if repo.git.diff("--staged"):
+            repo.git.commit(["-m", f"Add {singular} {path}"])
+
+
+@new_app.command(name="result")
+def new_result(
+    path: str,
+    title: Annotated[str, typer.Option("--title")],
+    description: Annotated[str | None, typer.Option("--description")] = None,
+    stage_name: Annotated[
+        str | None,
+        typer.Option(
+            "--stage",
+            help="Name of the pipeline stage that generates this result.",
+        ),
+    ] = None,
+    no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite existing result if one exists.",
+        ),
+    ] = False,
+):
+    """Declare a new result."""
+    _new_simple_artifact(
+        "results", path, title, description, stage_name, no_commit, overwrite
+    )
+
+
+@new_app.command(name="presentation|pres")
+def new_presentation(
+    path: str,
+    title: Annotated[str, typer.Option("--title")],
+    description: Annotated[str | None, typer.Option("--description")] = None,
+    stage_name: Annotated[
+        str | None,
+        typer.Option(
+            "--stage",
+            help="Name of the pipeline stage that generates this presentation.",
+        ),
+    ] = None,
+    no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite existing presentation if one exists.",
+        ),
+    ] = False,
+):
+    """Declare a new presentation."""
+    _new_simple_artifact(
+        "presentations",
+        path,
+        title,
+        description,
+        stage_name,
+        no_commit,
+        overwrite,
+    )
+
+
 @new_app.command("question")
 def new_question(
     question: str,

calkit/cli/scheduler.py

@@ -15,15 +15,17 @@ import shutil
 import signal
 import socket
 import subprocess
+import sys
 import time
 import uuid
+from typing import Any
 
 import typer
 from sqlitedict import SqliteDict
 from typing_extensions import Annotated
 
 import calkit
-from calkit.cli import AliasGroup, raise_error
+from calkit.cli import AliasGroup, raise_error, warn
 
 scheduler_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
@@ -75,6 +77,23 @@ def _record_job(name: str, info: dict) -> None:
         jobs[name] = info
 
 
+def _delete_job(name: str) -> None:
+    """Remove a job record from the database.
+
+    Used when a job is canceled (e.g. the user hits Ctrl+C while waiting for
+    it). Without this the record would linger, and the next ``calkit run``
+    would find the job gone from the queue with no exit status and wrongly
+    treat it as successful instead of resubmitting it.
+    """
+    if not os.path.isfile(JOBS_DB_PATH):
+        return
+    with SqliteDict(
+        JOBS_DB_PATH, autocommit=True, timeout=JOBS_DB_TIMEOUT
+    ) as jobs:
+        if name in jobs:
+            del jobs[name]
+
+
 def _mock_enabled() -> bool:
     """Whether scheduler commands should run jobs locally.
 
@@ -177,15 +196,20 @@ def _mock_submit(job_id: str, job_command: str, log_path: str) -> int:
     env = dict(os.environ)
     env.setdefault("PBS_O_WORKDIR", os.getcwd())
     with open(log_path, "w") as log_file:
-        popen_kwargs = dict(
+        # Annotated so the mixed value types don't defeat Popen's overloads.
+        popen_kwargs: dict[str, Any] = dict(
             stdout=log_file,
             stderr=subprocess.STDOUT,
             env=env,
         )
-        if os.name == "posix":
-            popen_kwargs["start_new_session"] = True
-        elif hasattr(subprocess, "CREATE_NEW_PROCESS_GROUP"):
+        # Detach the job into its own session/process group so it outlives
+        # this command and can be signaled as a group when canceled. Guard on
+        # `sys.platform` (not `os.name`) so the unused branch reads as
+        # platform-specific dead code rather than an unreachable bug.
+        if sys.platform == "win32":
             popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
+        else:
+            popen_kwargs["start_new_session"] = True
         proc = subprocess.Popen(
             ["bash", "-c", wrapped],
             **popen_kwargs,
@@ -213,31 +237,145 @@ def _mock_cancel(job_id: str) -> tuple[bool, str]:
     return True, ""
 
 
-def _is_active(kind: str, job_id: str) -> bool:
+def _mock_exit_code(job_id: str) -> int | None:
+    # The job writes its exit code to the status sentinel before exiting; a
+    # canceled job writes "canceled" instead, which isn't an integer.
+    try:
+        with open(_mock_status_path(job_id)) as f:
+            content = f.read().strip()
+    except OSError:
+        return None
+    try:
+        return int(content)
+    except ValueError:
+        return None
+
+
+def _parse_slurm_exit_code(value: str) -> int | None:
+    # SLURM reports exit status as "<code>:<signal>"; a job killed by a signal
+    # (e.g. out-of-memory, walltime) has a non-zero signal even if code is 0.
+    code_part, _, signal_part = value.strip().partition(":")
+    try:
+        code = int(code_part)
+        # An empty signal part means "no signal"; a non-empty one that won't
+        # parse is malformed, so report the whole value as unknown rather than
+        # silently treating it as a clean exit.
+        sig = int(signal_part) if signal_part else 0
+    except ValueError:
+        return None
+    return code if sig == 0 else 128 + sig
+
+
+def _slurm_exit_code(job_id: str) -> int | None:
+    # Prefer scontrol, which reports a finished job while it lingers in the
+    # controller's memory and is available even without accounting; fall back
+    # to sacct for jobs that have already aged out of scontrol.
+    p = subprocess.run(
+        ["scontrol", "show", "job", job_id],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if p.returncode == 0 and p.stdout.strip():
+        state = None
+        exit_code = None
+        for token in p.stdout.split():
+            if token.startswith("JobState="):
+                state = token.split("=", 1)[1]
+            elif token.startswith("ExitCode="):
+                exit_code = _parse_slurm_exit_code(token.split("=", 1)[1])
+        if state is not None:
+            if state == "COMPLETED":
+                return exit_code or 0
+            return exit_code or 1
+    p = subprocess.run(
+        ["sacct", "-j", job_id, "-n", "-P", "-o", "State,ExitCode"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if p.returncode == 0 and p.stdout.strip():
+        # The first line is the job itself (later lines are its steps).
+        state, _, code = p.stdout.strip().splitlines()[0].partition("|")
+        exit_code = _parse_slurm_exit_code(code)
+        if state.strip() == "COMPLETED":
+            return exit_code or 0
+        return exit_code or 1
+    return None
+
+
+def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
+    """Return ``(active, exit_code)`` for a scheduler job.
+
+    ``active`` is whether the job is still queued or running. ``exit_code`` is
+    the job's exit status once it has finished---0 for success, non-zero for
+    failure---or ``None`` while the job is still active or when the scheduler
+    no longer has a record from which to read it. A ``None`` exit code on a
+    finished job is an unknown outcome the caller must decide how to treat.
+    """
     if _mock_enabled():
-        return _mock_active(job_id)
+        if _mock_active(job_id):
+            return True, None
+        return False, _mock_exit_code(job_id)
     if kind == "slurm":
         p = subprocess.run(
-            ["squeue", "--job", job_id], capture_output=True, text=True
+            ["squeue", "--job", job_id],
+            capture_output=True,
+            text=True,
+            check=False,
         )
-        if p.returncode != 0:
-            return False
-        return len(p.stdout.strip().split("\n")) > 1
+        if p.returncode == 0:
+            # squeue prints a header row plus one row per matching job, so more
+            # than the header means the job is still queued or running.
+            if len(p.stdout.strip().split("\n")) > 1:
+                return True, None
+            return False, _slurm_exit_code(job_id)
+        # A non-zero exit is ambiguous like PBS: an invalid/unknown job id
+        # means the job is gone, but a transient controller failure should not
+        # end the wait early (nor let `_slurm_exit_code` call a still-running
+        # job failed), so only conclude the job is done when squeue positively
+        # reports the id is invalid; otherwise keep waiting.
+        stderr = (p.stderr or "").lower()
+        if "invalid job id" in stderr:
+            return False, _slurm_exit_code(job_id)
+        return True, None
     # Use `qstat -f` and parse job_state: on Torque/OpenPBS, plain `qstat
     # <id>` returns exit 0 even for completed (C) jobs, so checking the
     # return code alone would cause `calkit sched batch` to hang forever
-    # after a PBS job finishes.  States C and F mean the job is done.
+    # after a PBS job finishes.  States C and F mean the job is done, and a
+    # finished job's record carries its `Exit_status`.
     p = subprocess.run(
         ["qstat", "-f", job_id], capture_output=True, text=True, check=False
     )
     if p.returncode != 0:
-        return False
+        # A non-zero exit is ambiguous: either the job is gone (completed and
+        # purged from history) or `qstat` itself failed transiently---a busy
+        # PBS server periodically refuses connections or times out. Treating a
+        # transient failure as completion would stop the wait while the job is
+        # still running and writing its log, so only conclude the job is done
+        # when qstat positively reports it is unknown; otherwise keep waiting.
+        # A purged job (unknown to qstat) is done but with an unknowable exit
+        # status; any other error is transient, so report the job as active.
+        stderr = (p.stderr or "").lower()
+        return ("unknown job" not in stderr), None
+    state = None
+    exit_code = None
     for line in p.stdout.splitlines():
         stripped = line.strip()
         if stripped.startswith("job_state"):
             state = stripped.split("=", 1)[-1].strip()
-            return state not in ("C", "F")
-    return False
+        elif stripped.lower().startswith("exit_status"):
+            try:
+                exit_code = int(stripped.split("=", 1)[-1].strip())
+            except ValueError:
+                exit_code = None
+    if state in ("C", "F"):
+        return False, exit_code
+    return True, None
+
+
+def _is_active(kind: str, job_id: str) -> bool:
+    return _poll_job(kind, job_id)[0]
 
 
 def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
@@ -250,23 +388,93 @@ def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
     return p.returncode == 0, p.stderr
 
 
-def _wait_until_done(kind: str, job_id: str, name: str) -> None:
-    """Poll until the job finishes, canceling it on Ctrl+C.
+def _wait_until_done(kind: str, job_id: str, name: str) -> int | None:
+    """Poll until the job finishes, returning its exit code.
 
+    The exit code is the scheduler-reported status of the finished job (0 for
+    success, non-zero for failure), or ``None`` when it can't be determined.
     The job is submitted and tracked, so interrupting the local wait cancels
     the scheduler job before exiting rather than leaving it orphaned.
     """
     try:
-        while _is_active(kind, job_id):
+        while True:
+            active, exit_code = _poll_job(kind, job_id)
+            if not active:
+                return exit_code
             time.sleep(1)
     except KeyboardInterrupt:
         typer.echo(f"Interrupted; canceling job '{name}' ({job_id})")
         ok, stderr = _cancel(kind, job_id)
         if not ok:
             typer.echo(f"Failed to cancel job '{name}' ({job_id}): {stderr}")
+        # Drop the record so the next run resubmits this job instead of
+        # mistaking the canceled job (which has no exit status) for a success.
+        _delete_job(name)
         raise typer.Exit(130)
 
 
+def _wait_for_output_file(log_path: str, timeout: float = 120.0) -> None:
+    """Wait for a finished job's output log to be fully written.
+
+    A scheduler reports a job as done as soon as it leaves the queue, but the
+    job's ``-o`` log---which is the batch stage's declared DVC output---may not
+    be in place yet: PBS stages it back from the exec host's spool after the
+    job exits, and a job can still be flushing its final lines. Returning
+    before the file lands (or while it is still growing) makes the subsequent
+    ``dvc repro`` see a missing or mid-write output and wrongly report the job
+    as failed. Poll until the file exists and its size holds steady, or until
+    ``timeout`` elapses (then return and let DVC surface the real state).
+    """
+    deadline = time.monotonic() + timeout
+    last_size = -1
+    # Require the size to repeat across polls so we don't snapshot mid-write;
+    # a missing file reports -1, which never counts as stable.
+    stable_polls = 0
+    while time.monotonic() < deadline:
+        try:
+            size = os.path.getsize(log_path)
+        except OSError:
+            size = -1
+        if size >= 0 and size == last_size:
+            stable_polls += 1
+            if stable_polls >= 2:
+                return
+        else:
+            stable_polls = 0
+        last_size = size
+        time.sleep(1)
+
+
+def _finalize_job(
+    name: str, job_id: str, exit_code: int | None, log_path: str
+) -> None:
+    """Fail the command if the finished job did not succeed.
+
+    Waiting for the job to leave the queue only tells us it stopped, not
+    whether it worked, so a non-zero scheduler exit code is surfaced as an
+    error---raising a non-zero exit that propagates up through ``dvc repro``
+    so the stage is marked failed rather than silently recorded as done. An
+    unknown exit code (the scheduler purged the record before we could read
+    it) can't be judged, so we warn and let the stage's declared outputs be
+    the arbiter.
+    """
+    # Wait for the job's `-o` log---this stage's declared DVC output---to be
+    # staged back before we read from or point at it.
+    if not _mock_enabled():
+        _wait_for_output_file(log_path)
+    if exit_code is None:
+        warn(
+            f"Could not determine exit status for job '{name}' (ID {job_id}); "
+            f"assuming success. Check the log at {log_path}"
+        )
+        return
+    if exit_code != 0:
+        raise_error(
+            f"Job '{name}' (ID {job_id}) failed with exit code {exit_code}. "
+            f"See the log at {log_path}"
+        )
+
+
 @scheduler_app.command(name="batch")
 def run_batch(
     name: Annotated[
@@ -496,7 +704,11 @@ def run_batch(
         # The recorded job may have been submitted under a different
         # scheduler kind; use its own kind for activity/cancel checks.
         prev_kind = job_info.get("kind", kind)
-        running_or_queued = _is_active(prev_kind, job_id)
+        # Capture the exit code alongside liveness: if the job already left the
+        # queue (e.g. while we were disconnected) the scheduler may still have
+        # its status in history, and reading it now avoids losing it to a later
+        # purge.
+        running_or_queued, prev_exit_code = _poll_job(prev_kind, job_id)
         should_wait = True
 
         def _cancel_with_reason(reason: str) -> None:
@@ -543,14 +755,19 @@ def run_batch(
                     break
             if should_wait:
                 typer.echo("Waiting for job to finish")
-                _wait_until_done(prev_kind, job_id, name)
+                exit_code = _wait_until_done(prev_kind, job_id, name)
+                _finalize_job(name, job_id, exit_code, log_path)
                 raise typer.Exit(0)
         elif not os.environ.get("CALKIT_FORCE"):
-            # The job has left the queue. If nothing it depends on changed and
-            # its outputs are present, it already finished successfully (e.g.
-            # while the master process was disconnected), so treat it as done
-            # rather than resubmitting and discarding that work. Under --force
-            # (CALKIT_FORCE) we skip this check and resubmit.
+            # The job has left the queue (e.g. it finished while the master
+            # process was disconnected). If nothing it depends on changed and
+            # the prior submission succeeded---or its status was purged, in
+            # which case we assume success since there is no record left to
+            # judge by and the stage's declared outputs are the final
+            # arbiter---harvest it rather than resubmitting and discarding the
+            # work. A known failure instead falls through to resubmit, so
+            # `calkit run` retries it (e.g. after the user fixed the cause).
+            # Under --force (CALKIT_FORCE) we skip this and always resubmit.
             job_dep_md5s = job_info.get("dep_md5s", {})
             deps_unchanged = set(job_deps) == set(deps) and all(
                 current_dep_md5s.get(dep) == job_dep_md5s.get(dep)
@@ -561,13 +778,15 @@ def run_batch(
                 and job_args == args
                 and job_setup == setup_cmds
             )
-            outputs_present = bool(outs) and all(
-                os.path.exists(out) for out in outs
-            )
-            if deps_unchanged and command_unchanged and outputs_present:
+            if (
+                deps_unchanged
+                and command_unchanged
+                and prev_exit_code in (0, None)
+            ):
                 typer.echo(
-                    f"Job '{name}' already completed; skipping resubmission"
+                    f"Job '{name}' already left the queue; using its result"
                 )
+                _finalize_job(name, job_id, prev_exit_code, log_path)
                 raise typer.Exit(0)
     # Job is not running or queued, so we can submit. First, delete any
     # non-persistent outputs.
@@ -632,7 +851,8 @@ def run_batch(
         )
         raise typer.Exit(130)
     typer.echo("Waiting for job to finish")
-    _wait_until_done(kind, job_id, name)
+    exit_code = _wait_until_done(kind, job_id, name)
+    _finalize_job(name, job_id, exit_code, log_path)
 
 
 def _detect_interpreter(target: str) -> list[str]: