squarepeg 0.3.0__tar.gz → 0.4.1__tar.gz

{squarepeg-0.3.0 → squarepeg-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squarepeg
-Version: 0.3.0
+Version: 0.4.1
 Summary: Run a docker-run-style command as a Kubernetes Pod or Job, streaming its output like a local process
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -49,6 +49,9 @@ Full documentation lives in the [wiki](https://github.com/BioWilko/squarepeg/wik
 - [Troubleshooting](https://github.com/BioWilko/squarepeg/wiki/Troubleshooting)
 - [Development](https://github.com/BioWilko/squarepeg/wiki/Development)
 
+> **MicroK8s on Python 3.13+:** if connecting fails with `CA cert does not include key usage extension`,
+> set `lenient_tls: true` in your config — see [Troubleshooting](https://github.com/BioWilko/squarepeg/wiki/Troubleshooting#runner-errors-exit-code-125).
+
 See also [`examples/config.yaml`](examples/config.yaml) for a fully worked
 config file.
 

{squarepeg-0.3.0 → squarepeg-0.4.1}/README.md

@@ -35,6 +35,9 @@ Full documentation lives in the [wiki](https://github.com/BioWilko/squarepeg/wik
 - [Troubleshooting](https://github.com/BioWilko/squarepeg/wiki/Troubleshooting)
 - [Development](https://github.com/BioWilko/squarepeg/wiki/Development)
 
+> **MicroK8s on Python 3.13+:** if connecting fails with `CA cert does not include key usage extension`,
+> set `lenient_tls: true` in your config — see [Troubleshooting](https://github.com/BioWilko/squarepeg/wiki/Troubleshooting#runner-errors-exit-code-125).
+
 See also [`examples/config.yaml`](examples/config.yaml) for a fully worked
 config file.
 

{squarepeg-0.3.0 → squarepeg-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "squarepeg"
-version = "0.3.0"
+version = "0.4.1"
 description = "Run a docker-run-style command as a Kubernetes Pod or Job, streaming its output like a local process"
 readme = "README.md"
 requires-python = ">=3.11"

{squarepeg-0.3.0 → squarepeg-0.4.1}/squarepeg/cli.py

@@ -15,15 +15,16 @@ from squarepeg.dockerargs import (
     parse_env_entries,
     reject_unsupported,
 )
-from squarepeg.errors import SquarepegError, UsageError
+from squarepeg.errors import RunnerError, SquarepegError, UsageError
 from squarepeg.k8s.dryrun import server_dry_run
+from squarepeg.k8s.orphans import DEFAULT_MIN_AGE_SECONDS, clean_orphans
 from squarepeg.k8s.runner import run_manifest
 from squarepeg.k8s.session import Session
 from squarepeg.labels import RUN_ID_LABEL
 from squarepeg.log import chatter
 from squarepeg.manifest import build_job, build_pod
 from squarepeg.naming import generate_name, validate_rfc1123
-from squarepeg.quantities import docker_cpus_to_k8s, docker_memory_to_k8s
+from squarepeg.quantities import docker_cpus_to_k8s, docker_gpus_to_k8s, docker_memory_to_k8s
 from squarepeg.runspec import ResourceSpec, RunSpec
 from squarepeg.volumes import auto_mounts_from_config, parse_volume_flag
 
@@ -64,7 +65,6 @@ def _unsupported_callback(ctx, param, value):
 _UNSUPPORTED_GROUPS: list[tuple[list[str], bool]] = [
     (["--network"], False),
     (["--privileged"], True),
-    (["--gpus"], False),
     (["-p", "--publish"], False),
     (["-d", "--detach"], True),
     (["--restart"], False),
@@ -154,6 +154,7 @@ def cli():
 )
 @click.option("--cpus", "cpus", default=None)
 @click.option("-m", "--memory", "memory", default=None)
+@click.option("--gpus", "gpus", default=None, help="GPU count (sets both request and limit)")
 @click.option("--request-cpu", "request_cpu", default=None)
 @click.option("--limit-cpu", "limit_cpu", default=None)
 @click.option("--request-memory", "request_memory", default=None)
@@ -198,6 +199,7 @@ def run(
     pull,
     cpus,
     memory,
+    gpus,
     request_cpu,
     limit_cpu,
     request_memory,
@@ -300,6 +302,22 @@ def run(
     if cli_set_memory and limit_memory is None:
         claims.add(f"/spec/containers/[name={container_name}]/resources/limits/memory")
 
+    if "gpu_resource" in config_defaults:
+        resources.gpu_resource = str(config_defaults["gpu_resource"])
+    if "gpu_runtime_class" in config_defaults:
+        # null/"" disables it, for clusters where the NVIDIA runtime is the node default
+        resources.gpu_runtime_class = config_defaults["gpu_runtime_class"] or None
+    cli_set_gpus = gpus is not None
+    if gpus is None and "gpus" in config_defaults:
+        gpus = config_defaults["gpus"]
+    if gpus is not None:
+        resources.gpus = docker_gpus_to_k8s(gpus)
+    if cli_set_gpus:
+        for kind in ("requests", "limits"):
+            claims.add(
+                f"/spec/containers/[name={container_name}]/resources/{kind}/{resources.gpu_resource}"
+            )
+
     if name is not None:
         validate_rfc1123(name, what="--name")
         claims.add("/metadata/name")
@@ -336,6 +354,7 @@ def run(
         else coerce_int(resolved_config.get("timeout", 300), "timeout")
     )
     quiet = quiet or coerce_bool(resolved_config.get("quiet", False), "quiet")
+    lenient_tls = coerce_bool(resolved_config.get("lenient_tls", False), "lenient_tls")
     cleanup = (
         False if keep else coerce_bool(resolved_config.get("cleanup", True), "cleanup")
     )
@@ -343,7 +362,7 @@ def run(
         resolved_config.get("orphan_sweep", True), "orphan_sweep"
     )
     orphan_sweep_min_age = coerce_int(
-        resolved_config.get("orphan_sweep_min_age", 300), "orphan_sweep_min_age"
+        resolved_config.get("orphan_sweep_min_age", DEFAULT_MIN_AGE_SECONDS), "orphan_sweep_min_age"
     )
     if orphan_sweep_min_age < 0:
         raise UsageError(
@@ -417,7 +436,7 @@ def run(
         ui.set_run_tag(effective_run_id)
 
     if dry_run_server:
-        session = Session(namespace=spec.namespace, context=context, quiet=spec.quiet)
+        session = Session(namespace=spec.namespace, context=context, quiet=spec.quiet, lenient_tls=lenient_tls)
         spec.namespace = session.namespace
         validated = server_dry_run(session, spec, manifest)
         click.echo(yaml.safe_dump(validated, sort_keys=False), nl=False)
@@ -426,12 +445,104 @@ def run(
         click.echo(yaml.safe_dump(manifest, sort_keys=False), nl=False)
         return
 
-    session = Session(namespace=spec.namespace, context=context, quiet=spec.quiet)
+    session = Session(namespace=spec.namespace, context=context, quiet=spec.quiet, lenient_tls=lenient_tls)
     spec.namespace = session.namespace
     exit_code = run_manifest(session, spec, manifest)
     raise SystemExit(exit_code)
 
 
+def _format_age(seconds: float) -> str:
+    """kubectl-style compact duration: the two largest non-zero units, e.g. '10m12s',
+    '3h14m', '2d4h'. Used only for --dry-run's human-readable listing."""
+    seconds = int(seconds)
+    if seconds < 60:
+        return f"{seconds}s"
+    minutes, secs = divmod(seconds, 60)
+    if minutes < 60:
+        return f"{minutes}m{secs}s" if secs else f"{minutes}m"
+    hours, mins = divmod(minutes, 60)
+    if hours < 24:
+        return f"{hours}h{mins}m" if mins else f"{hours}h"
+    days, hrs = divmod(hours, 24)
+    return f"{days}d{hrs}h" if hrs else f"{days}d"
+
+
+def _candidate_detail(candidate) -> str:
+    age = _format_age(candidate.age_seconds)
+    if candidate.category == "terminal":
+        return f"finished {age} ago"
+    return f"{candidate.reason} for {age}"
+
+
+@cli.command("clean")
+@click.option("--dry-run", "dry_run", is_flag=True, help="list what would be deleted without deleting anything")
+@click.option(
+    "--min-age",
+    "min_age",
+    type=int,
+    default=None,
+    help="seconds a resource must be eligible before it's cleaned (default: config orphan_sweep_min_age, else 300)",
+)
+@click.option("-n", "--namespace", "namespace", default=None)
+@click.option("--context", "context", default=None, help="kubeconfig context to use")
+@click.option("--quiet", "quiet", is_flag=True)
+@click.option(
+    "--no-color",
+    "no_color",
+    is_flag=True,
+    help="disable colour/animation in squarepeg's own stderr output",
+)
+@_add_config_options
+def clean(dry_run, min_age, namespace, context, quiet, no_color, config_paths, no_default_config, profile):
+    """Delete this user's own orphaned Pods/Jobs left behind by crashed or killed runs.
+
+    Scoped to your own resources only (never a teammate's), and --keep'd resources are
+    always exempt -- there is no flag to override either. Runs regardless of the
+    'orphan_sweep' config key, which only affects the automatic per-run sweep. There is
+    no confirmation prompt: the candidates are, by construction, either already finished
+    or stuck and unable to start, so nothing running is ever at risk -- use --dry-run to
+    preview what would be deleted first if you want to check.
+    """
+    ui.set_color_override(False if no_color else None)
+    ui.set_run_tag(secrets.token_hex(4))
+
+    resolved_config, _sources = _resolve_config(config_paths, no_default_config, profile)
+
+    if namespace is None:
+        namespace = resolved_config.get("namespace")
+    quiet = quiet or coerce_bool(resolved_config.get("quiet", False), "quiet")
+    lenient_tls = coerce_bool(resolved_config.get("lenient_tls", False), "lenient_tls")
+
+    if min_age is None:
+        min_age = coerce_int(
+            resolved_config.get("orphan_sweep_min_age", DEFAULT_MIN_AGE_SECONDS), "orphan_sweep_min_age"
+        )
+        if min_age < 0:
+            raise UsageError(f"'orphan_sweep_min_age' must not be negative, got {min_age}")
+    elif min_age < 0:
+        raise UsageError(f"--min-age must not be negative, got {min_age}")
+
+    session = Session(namespace=namespace, context=context, quiet=quiet, lenient_tls=lenient_tls)
+
+    with ui.Status("scanning for orphaned resources", quiet=quiet):
+        result = clean_orphans(session, min_age, quiet=quiet, dry_run=dry_run)
+
+    if dry_run:
+        if result.candidates:
+            kind_name_width = max(len(f"{c.kind}/{c.name}") for c in result.candidates)
+            category_width = max(len(c.category) for c in result.candidates)
+            for c in result.candidates:
+                kind_name = f"{c.kind}/{c.name}".ljust(kind_name_width)
+                category = c.category.ljust(category_width)
+                click.echo(f"{kind_name}  {category}  {_candidate_detail(c)}")
+        return
+
+    if result.failed:
+        raise RunnerError(
+            f"failed to delete {len(result.failed)} of {len(result.candidates)} orphaned resource(s)"
+        )
+
+
 def _print_config(config_paths, no_default_config, profile):
     ui.set_run_tag(None)  # config has no run context, even if invoked right after a 'run' in-process
     resolved_config, sources = _resolve_config(config_paths, no_default_config, profile)

{squarepeg-0.3.0 → squarepeg-0.4.1}/squarepeg/config.py

@@ -31,6 +31,7 @@ ALLOWED_TOP_KEYS = {
     "timeout",
     "quiet",
     "allow_host_path_mounts",
+    "lenient_tls",
     "split_streams",
     "orphan_sweep",
     "orphan_sweep_min_age",
@@ -40,7 +41,16 @@ ALLOWED_TOP_KEYS = {
     "job",
     "profiles",
 }
-ALLOWED_DEFAULTS_KEYS = {"cpus", "memory", "image_pull_policy", "workdir", "env"}
+ALLOWED_DEFAULTS_KEYS = {
+    "cpus",
+    "memory",
+    "gpus",
+    "gpu_resource",
+    "gpu_runtime_class",
+    "image_pull_policy",
+    "workdir",
+    "env",
+}
 
 
 def _resolve_env_paths() -> list[tuple[Path, str]]:

{squarepeg-0.3.0 → squarepeg-0.4.1}/squarepeg/dockerargs.py

@@ -7,8 +7,6 @@ UNSUPPORTED: dict[str, str] = {
     "--network": "container networking is a cluster-level concept; use the 'kubernetes' passthrough "
     "section in your config (spec.dnsPolicy, spec.hostNetwork, etc.) if you need to control it",
     "--privileged": "use 'kubernetes.spec.containers[].securityContext.privileged' in your config",
-    "--gpus": "request GPUs via 'kubernetes.spec.containers[].resources' (e.g. 'nvidia.com/gpu: 1') "
-    "in your config",
     "-p": "there is no host to publish a port to on a remote cluster; use a Kubernetes Service instead",
     "--publish": "there is no host to publish a port to on a remote cluster; use a Kubernetes Service instead",
     "-d": "squarepeg always attaches to stream output; there is no detached mode",

squarepeg-0.4.1/squarepeg/k8s/events.py

@@ -0,0 +1,90 @@
+"""Surface pod events as diagnostics: the difference between 'it hung' and
+'0/12 nodes available: insufficient memory'.
+"""
+
+import threading
+
+from kubernetes.client.rest import ApiException
+
+from squarepeg.log import chatter
+
+# how often WarningEventWatcher re-lists events while waiting for a pod to start
+EVENT_POLL_SECONDS = 3
+
+
+def _list_events(session, name: str) -> list:
+    try:
+        events = session.core.list_namespaced_event(
+            session.namespace, field_selector=f"involvedObject.name={name}"
+        )
+    except ApiException:
+        return []  # best-effort diagnostics; never fail the run because event listing failed
+    return list(events.items or [])
+
+
+def _event_key(event) -> tuple[str, str]:
+    return (event.reason, event.message)
+
+
+def print_pod_events(
+    session, pod_name: str, *, quiet: bool = False, skip: frozenset[tuple[str, str]] = frozenset()
+) -> None:
+    """Print every event for `pod_name`, except those already shown (by (reason, message))."""
+    for event in _list_events(session, pod_name):
+        if _event_key(event) not in skip:
+            chatter(f"{pod_name}: {event.reason}: {event.message}", quiet=quiet, level="detail")
+
+
+class WarningEventWatcher:
+    """Context manager that, while squarepeg waits on `name` (a pod, or a job waiting to
+    create one), polls its Warning events in the background and prints each new one as
+    soon as it appears.
+
+    These are usually the real answer to "why isn't it starting?" -- an unschedulable pod,
+    a missing RuntimeClass, an exceeded quota, a volume that won't mount -- and without
+    this they'd only be printed once --timeout had already expired. squarepeg still waits
+    out the timeout rather than failing on them, since some do resolve on their own (a
+    cluster autoscaler adding a node, a PVC binding), and says so once.
+    """
+
+    def __init__(self, session, name: str, *, timeout: int, quiet: bool = False):
+        self.session = session
+        self.name = name
+        self.timeout = timeout
+        self.quiet = quiet
+        self.shown: set[tuple[str, str]] = set()
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def __enter__(self) -> "WarningEventWatcher":
+        if not self.quiet:
+            self._thread = threading.Thread(target=self._poll, daemon=True)
+            self._thread.start()
+        return self
+
+    def __exit__(self, *_exc) -> bool:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=2)
+        return False
+
+    def _poll(self) -> None:
+        while not self._stop.wait(EVENT_POLL_SECONDS):
+            self.report_new_warnings()
+
+    def report_new_warnings(self) -> None:
+        for event in _list_events(self.session, self.name):
+            key = _event_key(event)
+            if event.type != "Warning" or key in self.shown:
+                continue
+            first = not self.shown
+            self.shown.add(key)
+            chatter(f"{self.name}: {event.reason}: {event.message}", quiet=self.quiet, level="warn")
+            if first:
+                chatter(
+                    f"this may not resolve on its own, but squarepeg will keep waiting until the "
+                    f"{self.timeout}s --timeout in case it does (e.g. a node scaling up); "
+                    "Ctrl+C to give up now",
+                    quiet=self.quiet,
+                    level="detail",
+                )

squarepeg-0.4.1/squarepeg/k8s/orphans.py

@@ -0,0 +1,324 @@
+"""Opportunistic sweep for orphaned squarepeg-managed Pods/Jobs left behind by a
+previous invocation that crashed or was killed before it could clean up after
+itself (machine died, kill -9, network partition, ...).
+
+Runs alongside -- never instead of -- the per-run cleanup in runner.py, and is
+scoped strictly to the CURRENT user's own resources (never a teammate's, even
+in a shared namespace) via the server-side label selector.
+
+Two independent categories of orphan, because a resource that never managed to
+start looks nothing like one that finished and wasn't deleted:
+
+  - terminal:  Succeeded/Failed, older than the threshold since it finished.
+  - stuck:     Pending with a container waiting on a reason squarepeg already
+               treats as fatal-on-sight (see podstate.FAIL_FAST_REASONS -- the
+               same set runner.py uses to fail fast on ITS OWN pod). A bare
+               Pod has no backoffLimit/TTL, so a pod stuck like this (e.g. a
+               typo'd image reference) never reaches Succeeded/Failed and
+               would otherwise sit in the namespace forever.
+
+Never anything in Running/Pending-without-a-fail-fast-reason/Unknown, at any
+age -- that's the load-bearing safety property of the whole feature.
+
+`sweep_orphans` (used by runner.py's own per-run cleanup) and `clean_orphans`
+(used by the standalone `squarepeg clean` command) share the same discovery
+function but are deliberately separate orchestration loops: they differ in
+output verbosity, error-hint text, and failure-reporting contract, and forcing
+them into one function would need behaviour flags for what is otherwise a
+ten-line loop.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from kubernetes.client.rest import ApiException
+
+from squarepeg.errors import ApiError
+from squarepeg.k8s.apierrors import delete_resource, wrap_api_exception
+from squarepeg.k8s.podstate import (
+    TERMINAL_PHASES,
+    job_terminal_age_seconds,
+    pod_age_seconds,
+    stuck_waiting_reason,
+    terminal_age_seconds,
+)
+from squarepeg.labels import (
+    CREATED_BY_LABEL,
+    KEEP_LABEL,
+    MANAGED_BY_LABEL,
+    MANAGED_BY_VALUE,
+    RUN_ID_LABEL,
+    current_created_by,
+)
+from squarepeg.log import chatter
+
+DEFAULT_MIN_AGE_SECONDS = 300
+
+
+@dataclass(frozen=True)
+class OrphanCandidate:
+    kind: str  # "pod" | "job"
+    name: str
+    category: str  # "terminal" | "stuck"
+    age_seconds: float
+    reason: str | None = None  # the fail-fast waiting reason, for category == "stuck"
+
+
+@dataclass
+class CleanResult:
+    candidates: list[OrphanCandidate] = field(default_factory=list)
+    deleted: list[OrphanCandidate] = field(default_factory=list)
+    already_gone: list[OrphanCandidate] = field(default_factory=list)
+    failed: list[tuple[OrphanCandidate, str]] = field(default_factory=list)
+
+
+def _label_selector(created_by: str) -> str:
+    return f"{MANAGED_BY_LABEL}={MANAGED_BY_VALUE},{CREATED_BY_LABEL}={created_by},!{KEEP_LABEL}"
+
+
+def _job_name_for_pod(pod) -> str | None:
+    labels = pod.metadata.labels or {}
+    if "job-name" in labels:
+        return labels["job-name"]
+    for ref in pod.metadata.owner_references or []:
+        if ref.kind == "Job":
+            return ref.name
+    return None
+
+
+def _is_own_and_unexcluded(labels: dict, created_by: str, exclude_run_id: str | None) -> bool:
+    if labels.get(CREATED_BY_LABEL) != created_by:
+        return False
+    if exclude_run_id is not None and labels.get(RUN_ID_LABEL) == exclude_run_id:
+        return False
+    # belt-and-braces: the server-side selector already excludes anything carrying the keep
+    # label, but re-checking client-side means a selector typo can never turn into deleting
+    # a resource the user explicitly asked to keep.
+    if labels.get(KEEP_LABEL):
+        return False
+    return True
+
+
+def _pod_classification(pod, min_age_seconds: int, now: datetime) -> tuple[str, float, str | None] | None:
+    """(category, age_seconds, reason) for a pod old enough to sweep under either category,
+    or None if it doesn't qualify at all."""
+    phase = pod.status.phase
+    if phase in TERMINAL_PHASES:
+        age = terminal_age_seconds(pod, now)
+        if age is not None and age >= min_age_seconds:
+            return "terminal", age, None
+        return None
+    reason = stuck_waiting_reason(pod) if phase == "Pending" else None
+    if reason is not None:
+        age = pod_age_seconds(pod, now)
+        if age is not None and age >= min_age_seconds:
+            return "stuck", age, reason
+    return None
+
+
+def _find_pods(pods, created_by: str, min_age_seconds: int, exclude_run_id: str | None, now: datetime):
+    """Returns (bare_pod_candidates, stuck_child_job_info).
+
+    Job-owned pods are never deleted directly -- deleting a Job's child pod while the Job
+    still has a non-zero backoffLimit would make the controller spawn a replacement, i.e.
+    the sweep would start work rather than clean it up. Instead, a Job-owned pod that would
+    otherwise qualify is reported back so the caller can delete the parent Job instead,
+    which cascades to the pod via Background propagation.
+    """
+    bare_candidates: list[OrphanCandidate] = []
+    stuck_child_job_info: dict[str, tuple[float, str | None]] = {}
+
+    for pod in pods:
+        labels = pod.metadata.labels or {}
+        if not _is_own_and_unexcluded(labels, created_by, exclude_run_id):
+            continue
+        classification = _pod_classification(pod, min_age_seconds, now)
+        if classification is None:
+            continue
+        category, age, reason = classification
+
+        job_name = _job_name_for_pod(pod)
+        if job_name is not None:
+            stuck_child_job_info[job_name] = (age, reason)
+        else:
+            bare_candidates.append(OrphanCandidate("pod", pod.metadata.name, category, age, reason))
+
+    return bare_candidates, stuck_child_job_info
+
+
+def _find_jobs(
+    jobs,
+    created_by: str,
+    min_age_seconds: int,
+    exclude_run_id: str | None,
+    now: datetime,
+    stuck_child_job_info: dict,
+):
+    candidates = []
+    for job in jobs:
+        labels = job.metadata.labels or {}
+        if not _is_own_and_unexcluded(labels, created_by, exclude_run_id):
+            continue
+
+        name = job.metadata.name
+        age = job_terminal_age_seconds(job, now)
+        if age is not None and age >= min_age_seconds:
+            candidates.append(OrphanCandidate("job", name, "terminal", age))
+        elif name in stuck_child_job_info:
+            # no Complete/Failed condition yet (e.g. backoffLimit: 0 means a stuck pull
+            # never produces a failure condition at all), but its child pod has been
+            # stuck on a fail-fast reason long enough -- same verdict, different signal.
+            child_age, reason = stuck_child_job_info[name]
+            candidates.append(OrphanCandidate("job", name, "stuck", child_age, reason))
+    return candidates
+
+
+def find_orphan_candidates(
+    session,
+    created_by: str,
+    min_age_seconds: int,
+    *,
+    exclude_run_id: str | None = None,
+    now: datetime | None = None,
+) -> list[OrphanCandidate]:
+    """Discover this user's own orphaned Pods/Jobs, with classification detail.
+
+    Server-side scoped to app.kubernetes.io/managed-by=squarepeg, squarepeg.io/created-by
+    matching `created_by`, and NOT carrying squarepeg.io/keep -- so a teammate's resource or
+    a deliberately --keep'd one is never even returned by the apiserver. Client-side filtered
+    by phase/waiting-reason and age.
+
+    `now` defaults to the real current time; tests pass a fixed value.
+    """
+    now = now or datetime.now(timezone.utc)
+    selector = _label_selector(created_by)
+
+    pods = session.core.list_namespaced_pod(session.namespace, label_selector=selector).items
+    jobs = session.batch.list_namespaced_job(session.namespace, label_selector=selector).items
+
+    bare_pod_candidates, stuck_child_job_info = _find_pods(pods, created_by, min_age_seconds, exclude_run_id, now)
+    job_candidates = _find_jobs(jobs, created_by, min_age_seconds, exclude_run_id, now, stuck_child_job_info)
+    return bare_pod_candidates + job_candidates
+
+
+def find_orphaned_resources(
+    session,
+    created_by: str,
+    min_age_seconds: int,
+    *,
+    exclude_run_id: str | None = None,
+    now: datetime | None = None,
+) -> list[tuple[str, str]]:
+    """(kind, name) pairs only -- a thin wrapper over find_orphan_candidates for callers
+    that don't need the classification detail. Deletion is identical for every entry
+    regardless of which category matched."""
+    candidates = find_orphan_candidates(
+        session, created_by, min_age_seconds, exclude_run_id=exclude_run_id, now=now
+    )
+    return [(c.kind, c.name) for c in candidates]
+
+
+def sweep_orphans(
+    session,
+    min_age_seconds: int,
+    *,
+    quiet: bool = False,
+    exclude_run_id: str | None = None,
+    now: datetime | None = None,
+) -> int:
+    """Discover and delete this user's own orphaned resources, as a side effect of a run's
+    own cleanup (see runner.py's finally block).
+
+    Best-effort throughout: a per-resource delete failure is logged and skipped rather than
+    raised, so one undeletable orphan never aborts the rest of the sweep. The caller
+    (run_manifest) is responsible for making sure a failure here can never change the
+    current invocation's own exit code -- see runner.py's finally block.
+    """
+    created_by = current_created_by()
+    try:
+        candidates = find_orphaned_resources(
+            session, created_by, min_age_seconds, exclude_run_id=exclude_run_id, now=now
+        )
+    except ApiException as exc:
+        wrapped = wrap_api_exception(exc, "failed to list resources for orphan sweep")
+        raise ApiError(f"{wrapped} (set 'orphan_sweep: false' in config to disable)") from exc
+
+    deleted = 0
+    for kind, name in candidates:
+        try:
+            if delete_resource(session, kind, name):
+                deleted += 1
+        except Exception as exc:  # one bad delete must not abort the rest of the sweep
+            chatter(f"orphan sweep: failed to delete {kind} {name!r}: {exc}", quiet=quiet, level="warn")
+
+    if deleted:
+        chatter(f"swept {deleted} orphaned resource(s) from previous runs", quiet=quiet, level="success")
+    return deleted
+
+
+def clean_orphans(
+    session,
+    min_age_seconds: int,
+    *,
+    quiet: bool = False,
+    dry_run: bool = False,
+    now: datetime | None = None,
+) -> CleanResult:
+    """Discover and (unless dry_run) delete this user's own orphaned resources, for the
+    standalone `squarepeg clean` command.
+
+    Unlike sweep_orphans, per-resource failures are recorded in the returned CleanResult
+    rather than only logged -- clean has no run whose own exit code must stay independent
+    of sweep failures, so the caller (cli.py) uses the failure list to decide its own exit
+    code. Never excludes a "current run", since there is no current run to exclude.
+    """
+    created_by = current_created_by()
+    try:
+        candidates = find_orphan_candidates(session, created_by, min_age_seconds, now=now)
+    except ApiException as exc:
+        wrapped = wrap_api_exception(exc, "failed to list resources for cleanup")
+        raise ApiError(f"{wrapped} (the clean command needs 'list' on pods and jobs in the namespace)") from exc
+
+    result = CleanResult(candidates=candidates)
+    if not candidates:
+        chatter("nothing to clean", quiet=quiet, level="info")
+        return result
+    if dry_run:
+        chatter(
+            f"would delete {len(candidates)} orphaned resource(s) (dry run; nothing was deleted)",
+            quiet=quiet,
+            level="info",
+        )
+        return result
+
+    chatter(f"found {len(candidates)} orphaned resource(s)", quiet=quiet, level="info")
+    for candidate in candidates:
+        try:
+            if delete_resource(session, candidate.kind, candidate.name):
+                result.deleted.append(candidate)
+                chatter(
+                    f"deleted {candidate.kind} {candidate.name!r} "
+                    f"({candidate.category}, {candidate.age_seconds:.0f}s)",
+                    quiet=quiet,
+                    level="success",
+                )
+            else:
+                result.already_gone.append(candidate)
+                chatter(f"{candidate.kind} {candidate.name!r} was already gone", quiet=quiet, level="detail")
+        except Exception as exc:  # one bad delete must not abort the rest of the cleanup
+            # delete_resource's own ApiError message already reads "failed to delete
+            # {kind} {name!r}: ..." -- don't wrap it a second time.
+            result.failed.append((candidate, str(exc)))
+            chatter(str(exc), quiet=quiet, level="warn")
+
+    summary = f"cleaned {len(result.deleted)} of {len(candidates)} orphaned resource(s)"
+    extras = []
+    if result.already_gone:
+        extras.append(f"{len(result.already_gone)} already gone")
+    if result.failed:
+        extras.append(f"{len(result.failed)} failed")
+    if extras:
+        summary += f" ({', '.join(extras)})"
+    chatter(summary, quiet=quiet, level="warn" if result.failed else "success")
+
+    return result