calkit-python 0.41.11__tar.gz → 0.41.13__tar.gz

{calkit_python-0.41.11 → calkit_python-0.41.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.11
+Version: 0.41.13
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.11 → calkit_python-0.41.13}/calkit/cli/list.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 from typing import Annotated, Literal
 
 import typer
@@ -12,6 +13,32 @@ from calkit.cli import AliasGroup, raise_error, warn
 list_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
 
+def _echo_object(obj: dict) -> None:
+    """Print a single object in the human-readable YAML-ish listing format."""
+    # Copy so popping 'path' doesn't mutate the caller's dict.
+    obj = dict(obj)
+    path = obj.pop("path", None)
+    typer.echo(f"- path: {path}")
+    for k, v in obj.items():
+        if isinstance(v, dict):
+            typer.echo(f"    {k}:")
+            for k1, v1 in v.items():
+                typer.echo(f"      {k1}: {v1}")
+        elif isinstance(v, list):
+            typer.echo(f"    {k}:")
+            for item in v:
+                if isinstance(item, dict):
+                    for n, (k1, v1) in enumerate(item.items()):
+                        if n == 0:
+                            typer.echo(f"      - {k1}: {v1}")
+                        else:
+                            typer.echo(f"        {k1}: {v1}")
+                else:
+                    typer.echo(f"        - {item}")
+        else:
+            typer.echo(f"  {k}: {v}")
+
+
 def _list_objects(
     kind: Literal[
         "notebooks",
@@ -21,33 +48,44 @@ def _list_objects(
         "publications",
     ],
 ):
-    """List objects.
+    """List objects."""
+    ck_info = calkit.load_calkit_info()
+    for obj in ck_info.get(kind, []) or []:
+        _echo_object(obj)
+
 
-    TODO: This should probably just use some library to dump YAML to string.
+def _list_artifacts(
+    kind: Literal["figures", "datasets"],
+    json_output: bool,
+    declared_only: bool,
+):
+    """List figures or datasets, optionally including auto-detected ones.
+
+    By default, artifacts declared in ``calkit.yaml`` are merged with any
+    auto-detected from the project's files; ``--declared-only`` returns just the
+    declared ones. Each entry carries a ``detected`` flag so callers can tell
+    declared and auto-detected artifacts apart.
     """
     ck_info = calkit.load_calkit_info()
-    objects = ck_info.get(kind, [])
-    for obj in objects:
-        path = obj.pop("path")
-        typer.echo(f"- path: {path}")
-        for k, v in obj.items():
-            if isinstance(v, dict):
-                typer.echo(f"    {k}:")
-                for k1, v1 in v.items():
-                    typer.echo(f"      {k1}: {v1}")
-            elif isinstance(v, list):
-                typer.echo(f"    {k}:")
-                for item in v:
-                    if isinstance(item, dict):
-                        for n, (k1, v1) in enumerate(item.items()):
-                            if n == 0:
-                                typer.echo(f"      - {k1}: {v1}")
-                            else:
-                                typer.echo(f"        {k1}: {v1}")
-                    else:
-                        typer.echo(f"        - {item}")
-            else:
-                typer.echo(f"  {k}: {v}")
+    declared = ck_info.get(kind, []) or []
+    declared_paths = {o.get("path") for o in declared if isinstance(o, dict)}
+    detected: list[dict] = []
+    if not declared_only:
+        found = calkit.detect.detect_project_artifacts(ck_info=ck_info)
+        for path in found.get(kind, []):
+            if path not in declared_paths:
+                detected.append({"path": path})
+    if json_output:
+        result = [
+            {**o, "detected": False} for o in declared if isinstance(o, dict)
+        ]
+        result += [{**o, "detected": True} for o in detected]
+        typer.echo(json.dumps(result))
+        return
+    for obj in declared:
+        _echo_object(obj)
+    for obj in detected:
+        _echo_object({**obj, "detected": True})
 
 
 @list_app.command(name="notebooks|nb")
@@ -57,15 +95,43 @@ def list_notebooks():
 
 
 @list_app.command(name="figures|figs")
-def list_figures():
+def list_figures(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list figures declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
     """List figures in the project."""
-    _list_objects("figures")
+    _list_artifacts("figures", json_output, declared_only)
 
 
 @list_app.command(name="datasets")
-def list_datasets():
+def list_datasets(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list datasets declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
     """List datasets in the project."""
-    _list_objects("datasets")
+    _list_artifacts("datasets", json_output, declared_only)
 
 
 @list_app.command(name="publications|pubs")

{calkit_python-0.41.11 → calkit_python-0.41.13}/calkit/cli/main/core.py

@@ -1172,7 +1172,7 @@ def pull(
         and "-R" not in dvc_args
     ):
         dvc_args.append("--recursive")
-    result = calkit.dvc.run_dvc_command(["pull"] + dvc_args)
+    result = calkit.dvc.run_dvc_command(["pull"] + dvc_args, lock_retries=5)
     if result != 0:
         raise_error("DVC pull failed")
     calkit.dvc.zip.sync_all(direction="to-workspace")
@@ -1187,7 +1187,7 @@ def pull(
                 continue
             typer.echo(f"DVC pulling subproject: {sp_path}")
             sp_result = calkit.dvc.run_dvc_command(
-                ["pull"] + dvc_args, cwd=sp_path
+                ["pull"] + dvc_args, cwd=sp_path, lock_retries=5
             )
             if sp_result != 0:
                 raise_error(f"DVC pull failed for subproject: {sp_path}")
@@ -1224,7 +1224,9 @@ def push(
                     calkit.dvc.set_remote_auth(remote_name=name)
         if remotes:
             typer.echo("Pushing to DVC remote")
-            result = calkit.dvc.run_dvc_command(["push"] + dvc_args)
+            result = calkit.dvc.run_dvc_command(
+                ["push"] + dvc_args, lock_retries=5
+            )
             if result != 0:
                 raise_error("DVC push failed")
         else:

{calkit_python-0.41.11 → calkit_python-0.41.13}/calkit/cli/scheduler.py

@@ -15,15 +15,17 @@ import shutil
 import signal
 import socket
 import subprocess
+import sys
 import time
 import uuid
+from typing import Any
 
 import typer
 from sqlitedict import SqliteDict
 from typing_extensions import Annotated
 
 import calkit
-from calkit.cli import AliasGroup, raise_error
+from calkit.cli import AliasGroup, raise_error, warn
 
 scheduler_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
@@ -177,15 +179,20 @@ def _mock_submit(job_id: str, job_command: str, log_path: str) -> int:
     env = dict(os.environ)
     env.setdefault("PBS_O_WORKDIR", os.getcwd())
     with open(log_path, "w") as log_file:
-        popen_kwargs = dict(
+        # Annotated so the mixed value types don't defeat Popen's overloads.
+        popen_kwargs: dict[str, Any] = dict(
             stdout=log_file,
             stderr=subprocess.STDOUT,
             env=env,
         )
-        if os.name == "posix":
-            popen_kwargs["start_new_session"] = True
-        elif hasattr(subprocess, "CREATE_NEW_PROCESS_GROUP"):
+        # Detach the job into its own session/process group so it outlives
+        # this command and can be signaled as a group when canceled. Guard on
+        # `sys.platform` (not `os.name`) so the unused branch reads as
+        # platform-specific dead code rather than an unreachable bug.
+        if sys.platform == "win32":
             popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
+        else:
+            popen_kwargs["start_new_session"] = True
         proc = subprocess.Popen(
             ["bash", "-c", wrapped],
             **popen_kwargs,
@@ -213,31 +220,145 @@ def _mock_cancel(job_id: str) -> tuple[bool, str]:
     return True, ""
 
 
-def _is_active(kind: str, job_id: str) -> bool:
+def _mock_exit_code(job_id: str) -> int | None:
+    # The job writes its exit code to the status sentinel before exiting; a
+    # canceled job writes "canceled" instead, which isn't an integer.
+    try:
+        with open(_mock_status_path(job_id)) as f:
+            content = f.read().strip()
+    except OSError:
+        return None
+    try:
+        return int(content)
+    except ValueError:
+        return None
+
+
+def _parse_slurm_exit_code(value: str) -> int | None:
+    # SLURM reports exit status as "<code>:<signal>"; a job killed by a signal
+    # (e.g. out-of-memory, walltime) has a non-zero signal even if code is 0.
+    code_part, _, signal_part = value.strip().partition(":")
+    try:
+        code = int(code_part)
+        # An empty signal part means "no signal"; a non-empty one that won't
+        # parse is malformed, so report the whole value as unknown rather than
+        # silently treating it as a clean exit.
+        sig = int(signal_part) if signal_part else 0
+    except ValueError:
+        return None
+    return code if sig == 0 else 128 + sig
+
+
+def _slurm_exit_code(job_id: str) -> int | None:
+    # Prefer scontrol, which reports a finished job while it lingers in the
+    # controller's memory and is available even without accounting; fall back
+    # to sacct for jobs that have already aged out of scontrol.
+    p = subprocess.run(
+        ["scontrol", "show", "job", job_id],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if p.returncode == 0 and p.stdout.strip():
+        state = None
+        exit_code = None
+        for token in p.stdout.split():
+            if token.startswith("JobState="):
+                state = token.split("=", 1)[1]
+            elif token.startswith("ExitCode="):
+                exit_code = _parse_slurm_exit_code(token.split("=", 1)[1])
+        if state is not None:
+            if state == "COMPLETED":
+                return exit_code or 0
+            return exit_code or 1
+    p = subprocess.run(
+        ["sacct", "-j", job_id, "-n", "-P", "-o", "State,ExitCode"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if p.returncode == 0 and p.stdout.strip():
+        # The first line is the job itself (later lines are its steps).
+        state, _, code = p.stdout.strip().splitlines()[0].partition("|")
+        exit_code = _parse_slurm_exit_code(code)
+        if state.strip() == "COMPLETED":
+            return exit_code or 0
+        return exit_code or 1
+    return None
+
+
+def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
+    """Return ``(active, exit_code)`` for a scheduler job.
+
+    ``active`` is whether the job is still queued or running. ``exit_code`` is
+    the job's exit status once it has finished---0 for success, non-zero for
+    failure---or ``None`` while the job is still active or when the scheduler
+    no longer has a record from which to read it. A ``None`` exit code on a
+    finished job is an unknown outcome the caller must decide how to treat.
+    """
     if _mock_enabled():
-        return _mock_active(job_id)
+        if _mock_active(job_id):
+            return True, None
+        return False, _mock_exit_code(job_id)
     if kind == "slurm":
         p = subprocess.run(
-            ["squeue", "--job", job_id], capture_output=True, text=True
+            ["squeue", "--job", job_id],
+            capture_output=True,
+            text=True,
+            check=False,
         )
-        if p.returncode != 0:
-            return False
-        return len(p.stdout.strip().split("\n")) > 1
+        if p.returncode == 0:
+            # squeue prints a header row plus one row per matching job, so more
+            # than the header means the job is still queued or running.
+            if len(p.stdout.strip().split("\n")) > 1:
+                return True, None
+            return False, _slurm_exit_code(job_id)
+        # A non-zero exit is ambiguous like PBS: an invalid/unknown job id
+        # means the job is gone, but a transient controller failure should not
+        # end the wait early (nor let `_slurm_exit_code` call a still-running
+        # job failed), so only conclude the job is done when squeue positively
+        # reports the id is invalid; otherwise keep waiting.
+        stderr = (p.stderr or "").lower()
+        if "invalid job id" in stderr:
+            return False, _slurm_exit_code(job_id)
+        return True, None
     # Use `qstat -f` and parse job_state: on Torque/OpenPBS, plain `qstat
     # <id>` returns exit 0 even for completed (C) jobs, so checking the
     # return code alone would cause `calkit sched batch` to hang forever
-    # after a PBS job finishes.  States C and F mean the job is done.
+    # after a PBS job finishes.  States C and F mean the job is done, and a
+    # finished job's record carries its `Exit_status`.
     p = subprocess.run(
         ["qstat", "-f", job_id], capture_output=True, text=True, check=False
     )
     if p.returncode != 0:
-        return False
+        # A non-zero exit is ambiguous: either the job is gone (completed and
+        # purged from history) or `qstat` itself failed transiently---a busy
+        # PBS server periodically refuses connections or times out. Treating a
+        # transient failure as completion would stop the wait while the job is
+        # still running and writing its log, so only conclude the job is done
+        # when qstat positively reports it is unknown; otherwise keep waiting.
+        # A purged job (unknown to qstat) is done but with an unknowable exit
+        # status; any other error is transient, so report the job as active.
+        stderr = (p.stderr or "").lower()
+        return ("unknown job" not in stderr), None
+    state = None
+    exit_code = None
     for line in p.stdout.splitlines():
         stripped = line.strip()
         if stripped.startswith("job_state"):
             state = stripped.split("=", 1)[-1].strip()
-            return state not in ("C", "F")
-    return False
+        elif stripped.lower().startswith("exit_status"):
+            try:
+                exit_code = int(stripped.split("=", 1)[-1].strip())
+            except ValueError:
+                exit_code = None
+    if state in ("C", "F"):
+        return False, exit_code
+    return True, None
+
+
+def _is_active(kind: str, job_id: str) -> bool:
+    return _poll_job(kind, job_id)[0]
 
 
 def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
@@ -250,14 +371,19 @@ def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
     return p.returncode == 0, p.stderr
 
 
-def _wait_until_done(kind: str, job_id: str, name: str) -> None:
-    """Poll until the job finishes, canceling it on Ctrl+C.
+def _wait_until_done(kind: str, job_id: str, name: str) -> int | None:
+    """Poll until the job finishes, returning its exit code.
 
+    The exit code is the scheduler-reported status of the finished job (0 for
+    success, non-zero for failure), or ``None`` when it can't be determined.
     The job is submitted and tracked, so interrupting the local wait cancels
     the scheduler job before exiting rather than leaving it orphaned.
     """
     try:
-        while _is_active(kind, job_id):
+        while True:
+            active, exit_code = _poll_job(kind, job_id)
+            if not active:
+                return exit_code
             time.sleep(1)
     except KeyboardInterrupt:
         typer.echo(f"Interrupted; canceling job '{name}' ({job_id})")
@@ -267,6 +393,68 @@ def _wait_until_done(kind: str, job_id: str, name: str) -> None:
         raise typer.Exit(130)
 
 
+def _wait_for_output_file(log_path: str, timeout: float = 120.0) -> None:
+    """Wait for a finished job's output log to be fully written.
+
+    A scheduler reports a job as done as soon as it leaves the queue, but the
+    job's ``-o`` log---which is the batch stage's declared DVC output---may not
+    be in place yet: PBS stages it back from the exec host's spool after the
+    job exits, and a job can still be flushing its final lines. Returning
+    before the file lands (or while it is still growing) makes the subsequent
+    ``dvc repro`` see a missing or mid-write output and wrongly report the job
+    as failed. Poll until the file exists and its size holds steady, or until
+    ``timeout`` elapses (then return and let DVC surface the real state).
+    """
+    deadline = time.monotonic() + timeout
+    last_size = -1
+    # Require the size to repeat across polls so we don't snapshot mid-write;
+    # a missing file reports -1, which never counts as stable.
+    stable_polls = 0
+    while time.monotonic() < deadline:
+        try:
+            size = os.path.getsize(log_path)
+        except OSError:
+            size = -1
+        if size >= 0 and size == last_size:
+            stable_polls += 1
+            if stable_polls >= 2:
+                return
+        else:
+            stable_polls = 0
+        last_size = size
+        time.sleep(1)
+
+
+def _finalize_job(
+    name: str, job_id: str, exit_code: int | None, log_path: str
+) -> None:
+    """Fail the command if the finished job did not succeed.
+
+    Waiting for the job to leave the queue only tells us it stopped, not
+    whether it worked, so a non-zero scheduler exit code is surfaced as an
+    error---raising a non-zero exit that propagates up through ``dvc repro``
+    so the stage is marked failed rather than silently recorded as done. An
+    unknown exit code (the scheduler purged the record before we could read
+    it) can't be judged, so we warn and let the stage's declared outputs be
+    the arbiter.
+    """
+    # Wait for the job's `-o` log---this stage's declared DVC output---to be
+    # staged back before we read from or point at it.
+    if not _mock_enabled():
+        _wait_for_output_file(log_path)
+    if exit_code is None:
+        warn(
+            f"Could not determine exit status for job '{name}' (ID {job_id}); "
+            f"assuming success. Check the log at {log_path}"
+        )
+        return
+    if exit_code != 0:
+        raise_error(
+            f"Job '{name}' (ID {job_id}) failed with exit code {exit_code}. "
+            f"See the log at {log_path}"
+        )
+
+
 @scheduler_app.command(name="batch")
 def run_batch(
     name: Annotated[
@@ -496,7 +684,11 @@ def run_batch(
         # The recorded job may have been submitted under a different
         # scheduler kind; use its own kind for activity/cancel checks.
         prev_kind = job_info.get("kind", kind)
-        running_or_queued = _is_active(prev_kind, job_id)
+        # Capture the exit code alongside liveness: if the job already left the
+        # queue (e.g. while we were disconnected) the scheduler may still have
+        # its status in history, and reading it now avoids losing it to a later
+        # purge.
+        running_or_queued, prev_exit_code = _poll_job(prev_kind, job_id)
         should_wait = True
 
         def _cancel_with_reason(reason: str) -> None:
@@ -543,14 +735,19 @@ def run_batch(
                     break
             if should_wait:
                 typer.echo("Waiting for job to finish")
-                _wait_until_done(prev_kind, job_id, name)
+                exit_code = _wait_until_done(prev_kind, job_id, name)
+                _finalize_job(name, job_id, exit_code, log_path)
                 raise typer.Exit(0)
         elif not os.environ.get("CALKIT_FORCE"):
-            # The job has left the queue. If nothing it depends on changed and
-            # its outputs are present, it already finished successfully (e.g.
-            # while the master process was disconnected), so treat it as done
-            # rather than resubmitting and discarding that work. Under --force
-            # (CALKIT_FORCE) we skip this check and resubmit.
+            # The job has left the queue (e.g. it finished while the master
+            # process was disconnected). If nothing it depends on changed and
+            # the prior submission succeeded---or its status was purged, in
+            # which case we assume success since there is no record left to
+            # judge by and the stage's declared outputs are the final
+            # arbiter---harvest it rather than resubmitting and discarding the
+            # work. A known failure instead falls through to resubmit, so
+            # `calkit run` retries it (e.g. after the user fixed the cause).
+            # Under --force (CALKIT_FORCE) we skip this and always resubmit.
             job_dep_md5s = job_info.get("dep_md5s", {})
             deps_unchanged = set(job_deps) == set(deps) and all(
                 current_dep_md5s.get(dep) == job_dep_md5s.get(dep)
@@ -561,13 +758,15 @@ def run_batch(
                 and job_args == args
                 and job_setup == setup_cmds
             )
-            outputs_present = bool(outs) and all(
-                os.path.exists(out) for out in outs
-            )
-            if deps_unchanged and command_unchanged and outputs_present:
+            if (
+                deps_unchanged
+                and command_unchanged
+                and prev_exit_code in (0, None)
+            ):
                 typer.echo(
-                    f"Job '{name}' already completed; skipping resubmission"
+                    f"Job '{name}' already left the queue; using its result"
                 )
+                _finalize_job(name, job_id, prev_exit_code, log_path)
                 raise typer.Exit(0)
     # Job is not running or queued, so we can submit. First, delete any
     # non-persistent outputs.
@@ -632,7 +831,8 @@ def run_batch(
         )
         raise typer.Exit(130)
     typer.echo("Waiting for job to finish")
-    _wait_until_done(kind, job_id, name)
+    exit_code = _wait_until_done(kind, job_id, name)
+    _finalize_job(name, job_id, exit_code, log_path)
 
 
 def _detect_interpreter(target: str) -> list[str]:

{calkit_python-0.41.11 → calkit_python-0.41.13}/calkit/cloud.py

@@ -15,7 +15,8 @@ from functools import partial
 from typing import Literal
 
 import requests
-from requests.exceptions import HTTPError
+from requests.exceptions import ConnectionError as RequestsConnectionError
+from requests.exceptions import HTTPError, Timeout
 
 from . import config
 
@@ -104,8 +105,7 @@ def run_device_flow() -> str:
                     ) from e
                 if "Device code not found" in txt:
                     raise DeviceLoginError(
-                        "Device code not found; "
-                        "Run 'calkit cloud login' again"
+                        "Device code not found; Run 'calkit cloud login' again"
                     ) from e
                 raise DeviceLoginError(
                     f"Error while polling for device authorization: {e}"
@@ -304,6 +304,9 @@ def _request(
     max_retries = 10
     base_delay_seconds = 0.25
     max_delay_seconds = 30
+    # Bound how long a single attempt can hang so stalled connections become
+    # retryable timeouts rather than blocking forever. Callers can override.
+    kwargs.setdefault("timeout", (10, 120))
     func = getattr(requests, kind)
     if base_url is None:
         base_url = get_base_url()
@@ -333,15 +336,29 @@ def _request(
                 run_device_flow()
                 continue
             raise
-        resp = func(
-            base_url + path,
-            params=params,
-            json=json,
-            data=data,
-            headers=req_headers,
-            **kwargs,
-        )
-        if resp.status_code == 502 and retry_num < max_retries:
+        try:
+            resp = func(
+                base_url + path,
+                params=params,
+                json=json,
+                data=data,
+                headers=req_headers,
+                **kwargs,
+            )
+        except (Timeout, RequestsConnectionError):
+            # Transient network failure (read/connect timeout, connection
+            # reset/refused). Retry with the same backoff as transient 5xx
+            # responses; re-raise once retries are exhausted.
+            if retry_num >= max_retries:
+                raise
+            wait = min(base_delay_seconds * (2**retry_num), max_delay_seconds)
+            time.sleep(wait)
+            continue
+        # Retry transient server-side errors (bad gateway, overloaded,
+        # unavailable, gateway timeout) with exponential backoff. These are
+        # typically infrastructure hiccups that resolve on a retry.
+        is_transient_5xx = resp.status_code in (500, 502, 503, 504)
+        if is_transient_5xx and retry_num < max_retries:
             wait = min(base_delay_seconds * (2**retry_num), max_delay_seconds)
             time.sleep(wait)
             continue