localqueue 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

localqueue/cli.py

@@ -7,6 +7,7 @@ import signal
 import subprocess
 import sys
 import time
+from collections import Counter
 from collections.abc import Callable
 from contextlib import contextmanager
 from dataclasses import dataclass
@@ -17,6 +18,7 @@ from typing import Any, Iterator
 
 from tenacity import wait_none
 
+from .failure import is_permanent_failure
 from .queue import PersistentQueue
 from .retry import (
     PersistentRetryExhausted,
@@ -24,6 +26,13 @@ from .retry import (
     SQLiteAttemptStore,
 )
 from .store import QueueMessage
+from .worker import (
+    PersistentWorkerConfig,
+    WorkerPolicyState,
+    _record_failure,
+    _record_success,
+    _sleep_for_policy,
+)
 
 DEFAULT_STORE_PATH = "localqueue_queue.sqlite3"
 DEFAULT_RETRY_STORE_PATH = "localqueue_retries.sqlite3"
@@ -34,6 +43,8 @@ CONFIG_FILENAME = "config.yaml"
 class _ProcessResult:
     processed: bool
     last_error: dict[str, Any] | None = None
+    final_state: str | None = None
+    permanent_failure: bool = False
 
 
 @dataclass(slots=True)
@@ -97,8 +108,10 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     config = _load_config(yaml)
     app = typer.Typer(no_args_is_help=True)
     queue_app = typer.Typer(no_args_is_help=True)
+    retry_app = typer.Typer(no_args_is_help=True)
     config_app = typer.Typer(no_args_is_help=True)
     app.add_typer(queue_app, name="queue")
+    app.add_typer(retry_app, name="retry")
     app.add_typer(config_app, name="config")
 
     @config_app.command("path")
@@ -113,6 +126,12 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     def config_init(
         store_path: str = typer.Option(DEFAULT_STORE_PATH, "--store-path"),
         retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
+        dead_letter_ttl_seconds: float | None = typer.Option(
+            None, "--dead-letter-ttl-seconds", min=0.0
+        ),
+        retry_record_ttl_seconds: float | None = typer.Option(
+            None, "--retry-record-ttl-seconds", min=0.0
+        ),
         force: bool = typer.Option(False, "--force"),
     ) -> None:
         path = _config_path()
@@ -125,6 +144,10 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         new_config: dict[str, Any] = {"store_path": store_path}
         if retry_store_path is not None:
             new_config["retry_store_path"] = retry_store_path
+        if dead_letter_ttl_seconds is not None:
+            new_config["dead_letter_ttl_seconds"] = dead_letter_ttl_seconds
+        if retry_record_ttl_seconds is not None:
+            new_config["retry_record_ttl_seconds"] = retry_record_ttl_seconds
         _write_config(yaml, new_config)
         config.clear()
         config.update(new_config)
@@ -135,15 +158,23 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         key: str,
         value: str,
     ) -> None:
-        if key not in {"store_path", "retry_store_path"}:
+        if key not in {
+            "store_path",
+            "retry_store_path",
+            "dead_letter_ttl_seconds",
+            "retry_record_ttl_seconds",
+        }:
             err_console.print(
                 "[red]unsupported config key:[/red] "
-                + f"{key}; use store_path or retry_store_path"
+                + (
+                    f"{key}; use store_path, retry_store_path, "
+                    "dead_letter_ttl_seconds, or retry_record_ttl_seconds"
+                )
             )
             raise typer.Exit(1)
 
         updated = dict(config)
-        updated[key] = value
+        updated[key] = _coerce_config_value(key, value)
         _write_config(yaml, updated)
         config.clear()
         config.update(updated)
@@ -155,7 +186,9 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         value: str | None = typer.Option(None, "--value"),
         store_path: str | None = typer.Option(None, "--store-path"),
         delay: float = typer.Option(0.0, "--delay", min=0.0),
+        dedupe_key: str | None = typer.Option(None, "--dedupe-key"),
         raw: bool = typer.Option(False, "--raw"),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
         try:
             payload_value = _read_value(value)
@@ -163,8 +196,17 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         except ValueError as exc:
             raise typer.BadParameter(str(exc)) from exc
         message = _queue(queue, _resolve_store_path(store_path, config)).put(
-            payload, delay=delay
+            payload, delay=delay, dedupe_key=dedupe_key
         )
+        if log_events:
+            _emit_event(
+                err_console,
+                "queue.enqueue",
+                queue=queue,
+                message_id=message.id,
+                state=message.state,
+                available_at=message.available_at,
+            )
         _print_json(console, _message_payload(message))
 
     @queue_app.command("pop")
@@ -175,6 +217,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         worker_id: str | None = typer.Option(None, "--worker-id"),
         block: bool = typer.Option(False, "--block"),
         timeout: float | None = typer.Option(None, "--timeout", min=0.0),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
         try:
             message = _queue(
@@ -185,6 +228,16 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         except Empty as exc:
             err_console.print("[yellow]queue is empty[/yellow]")
             raise typer.Exit(1) from exc
+        if log_events:
+            _emit_event(
+                err_console,
+                "queue.lease",
+                queue=message.queue,
+                message_id=message.id,
+                leased_by=message.leased_by,
+                attempts=message.attempts,
+                leased_until=message.leased_until,
+            )
         _print_json(console, _message_payload(message))
 
     @queue_app.command("inspect")
@@ -192,6 +245,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         queue: str,
         message_id: str,
         store_path: str | None = typer.Option(None, "--store-path"),
+        json_output: bool = typer.Option(False, "--json"),
     ) -> None:
         message = _queue(queue, _resolve_store_path(store_path, config)).inspect(
             message_id
@@ -206,6 +260,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         queue: str,
         message_id: str,
         store_path: str | None = typer.Option(None, "--store-path"),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
         _complete_message(
             typer,
@@ -215,6 +270,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             _resolve_store_path(store_path, config),
             message_id,
             "ack",
+            log_events=log_events,
         )
 
     @queue_app.command("release")
@@ -223,6 +279,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         message_id: str,
         store_path: str | None = typer.Option(None, "--store-path"),
         delay: float = typer.Option(0.0, "--delay", min=0.0),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
         _complete_message(
             typer,
@@ -233,6 +290,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             message_id,
             "release",
             delay=delay,
+            log_events=log_events,
         )
 
     @queue_app.command("dead-letter")
@@ -240,6 +298,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         queue: str,
         message_id: str,
         store_path: str | None = typer.Option(None, "--store-path"),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
         _complete_message(
             typer,
@@ -249,6 +308,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             _resolve_store_path(store_path, config),
             message_id,
             "dead-letter",
+            log_events=log_events,
         )
 
     @queue_app.command("size")
@@ -264,6 +324,8 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         store_path: str | None = typer.Option(None, "--store-path"),
         watch: bool = typer.Option(False, "--watch"),
         interval: float = typer.Option(1.0, "--interval", min=0.001),
+        stale_after: float | None = typer.Option(None, "--stale-after", min=0.0),
+        json_output: bool = typer.Option(False, "--json"),
     ) -> None:
         persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
         with _shutdown_state() as shutdown:
@@ -272,9 +334,33 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 console=console,
                 watch=watch,
                 interval=interval,
+                stale_after=stale_after,
                 shutdown=shutdown,
             )
 
+    @queue_app.command("health")
+    def queue_health(
+        queue: str,
+        store_path: str | None = typer.Option(None, "--store-path"),
+        stale_after: float = typer.Option(120.0, "--stale-after", min=0.0),
+        json_output: bool = typer.Option(False, "--json"),
+    ) -> None:
+        persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
+        stats = persistent_queue.stats()
+        worker_health = _worker_health_summary(
+            stats.last_seen_by_worker_id, stale_after=stale_after
+        )
+        _print_json(
+            console,
+            {
+                "queue": queue,
+                "stats": stats.as_dict(),
+                "worker_health": worker_health,
+                "dead_letters": _dead_letter_summary(persistent_queue.dead_letters()),
+                "retention": _retention_settings(config),
+            },
+        )
+
     @queue_app.command("dead")
     def queue_dead(
         queue: str,
@@ -282,8 +368,56 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         limit: int | None = typer.Option(None, "--limit", min=0),
         watch: bool = typer.Option(False, "--watch"),
         interval: float = typer.Option(1.0, "--interval", min=0.001),
+        json_output: bool = typer.Option(False, "--json"),
+        summary: bool = typer.Option(False, "--summary"),
+        min_attempts: int | None = typer.Option(None, "--min-attempts", min=0),
+        max_attempts: int | None = typer.Option(None, "--max-attempts", min=0),
+        error_contains: str | None = typer.Option(None, "--error-contains"),
+        failed_within: float | None = typer.Option(None, "--failed-within", min=0.0),
+        dry_run: bool = typer.Option(False, "--dry-run"),
+        prune_older_than: float | None = typer.Option(
+            None, "--prune-older-than", min=0.0
+        ),
     ) -> None:
         persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
+        if prune_older_than is not None or dry_run:
+            older_than = _resolve_dead_letter_ttl(prune_older_than, config)
+            if older_than is None:
+                err_console.print(
+                    "[red]pass --prune-older-than or configure "
+                    "dead_letter_ttl_seconds[/red]"
+                )
+                raise typer.Exit(1)
+            if any(
+                (
+                    watch,
+                    summary,
+                    limit is not None,
+                    min_attempts is not None,
+                    max_attempts is not None,
+                    error_contains is not None,
+                    failed_within is not None,
+                )
+            ):
+                err_console.print(
+                    "[red]pass --prune-older-than on its own; "
+                    + "it cannot be combined with list or watch options[/red]"
+                )
+                raise typer.Exit(1)
+            deleted = (
+                persistent_queue.count_dead_letters_older_than(older_than=older_than)
+                if dry_run
+                else persistent_queue.prune_dead_letters(older_than=older_than)
+            )
+            _print_json(
+                console,
+                {
+                    "dry_run": dry_run,
+                    "older_than": older_than,
+                    "would_delete" if dry_run else "deleted": deleted,
+                },
+            )
+            return
         with _shutdown_state() as shutdown:
             _print_dead_letters(
                 persistent_queue,
@@ -292,7 +426,45 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 watch=watch,
                 interval=interval,
                 shutdown=shutdown,
+                summary=summary,
+                min_attempts=min_attempts,
+                max_attempts=max_attempts,
+                error_contains=error_contains,
+                failed_within=failed_within,
+            )
+
+    @retry_app.command("prune")
+    def retry_prune(
+        older_than: float | None = typer.Option(None, "--older-than", min=0.0),
+        retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
+        dry_run: bool = typer.Option(False, "--dry-run"),
+    ) -> None:
+        store = SQLiteAttemptStore(_resolve_retry_store_path(retry_store_path, config))
+        try:
+            resolved_older_than = _resolve_retry_record_ttl(older_than, config)
+            if resolved_older_than is None:
+                err_console.print(
+                    "[red]pass --older-than or configure retry_record_ttl_seconds[/red]"
+                )
+                raise typer.Exit(1)
+            now = time.time()
+            deleted = (
+                store.count_exhausted_older_than(
+                    older_than=resolved_older_than, now=now
+                )
+                if dry_run
+                else store.prune_exhausted(older_than=resolved_older_than, now=now)
             )
+        finally:
+            store.close()
+        _print_json(
+            console,
+            {
+                "dry_run": dry_run,
+                "older_than": resolved_older_than,
+                "would_delete" if dry_run else "deleted": deleted,
+            },
+        )
 
     @queue_app.command("requeue-dead")
     def queue_requeue_dead(
@@ -301,6 +473,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         store_path: str | None = typer.Option(None, "--store-path"),
         delay: float = typer.Option(0.0, "--delay", min=0.0),
         all_: bool = typer.Option(False, "--all"),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
         persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
         if all_:
@@ -314,6 +487,15 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             for message in messages:
                 if persistent_queue.requeue_dead(message, delay=delay):
                     requeued += 1
+            if log_events:
+                _emit_event(
+                    err_console,
+                    "queue.requeue",
+                    queue=queue,
+                    all=True,
+                    requeued=requeued,
+                    delay=delay,
+                )
             _print_json(console, {"requeued": requeued})
             return
         if message_id is None:
@@ -325,14 +507,26 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         if not persistent_queue.requeue_dead(message, delay=delay):
             err_console.print(f"[red]dead-letter message not found:[/red] {message_id}")
             raise typer.Exit(1)
+        if log_events:
+            _emit_event(
+                err_console,
+                "queue.requeue",
+                queue=queue,
+                message_id=message_id,
+                delay=delay,
+            )
         _print_json(console, {"id": message_id, "state": "requeued"})
 
     @queue_app.command("purge")
     def queue_purge(
         queue: str,
         store_path: str | None = typer.Option(None, "--store-path"),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
-        console.print(_queue(queue, _resolve_store_path(store_path, config)).purge())
+        deleted = _queue(queue, _resolve_store_path(store_path, config)).purge()
+        if log_events:
+            _emit_event(err_console, "queue.purge", queue=queue, deleted=deleted)
+        console.print(deleted)
 
     @queue_app.command("process")
     def queue_process(
@@ -349,11 +543,27 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         timeout: float | None = typer.Option(None, "--timeout", min=0.0),
         idle_sleep: float = typer.Option(1.0, "--idle-sleep", min=0.001),
         release_delay: float = typer.Option(0.0, "--release-delay", min=0.0),
+        min_interval: float = typer.Option(0.0, "--min-interval", min=0.0),
+        circuit_breaker_failures: int = typer.Option(
+            0, "--circuit-breaker-failures", min=0
+        ),
+        circuit_breaker_cooldown: float = typer.Option(
+            0.0, "--circuit-breaker-cooldown", min=0.0
+        ),
         dead_letter_on_exhaustion: bool = typer.Option(
             True,
             "--dead-letter-on-exhaustion/--release-on-exhaustion",
         ),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
+        try:
+            _validate_worker_loop_options(
+                max_jobs=max_jobs,
+                forever=forever,
+            )
+        except ValueError as exc:
+            err_console.print(f"[red]{exc}[/red]")
+            raise typer.Exit(1) from exc
         try:
             worker = _load_callable(handler)
         except (AttributeError, ImportError, TypeError, ValueError) as exc:
@@ -364,6 +574,13 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             _resolve_store_path(store_path, config),
             lease_timeout=lease_timeout,
         )
+        if worker_id is not None:
+            persistent_queue.record_worker_heartbeat(worker_id)
+        worker_policy = PersistentWorkerConfig(
+            min_interval=min_interval,
+            circuit_breaker_failures=circuit_breaker_failures,
+            circuit_breaker_cooldown=circuit_breaker_cooldown,
+        )
         with _shutdown_state() as shutdown:
             exit_code = _process_queue_messages(
                 persistent_queue,
@@ -371,6 +588,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 console=console,
                 err_console=err_console,
                 shutdown=shutdown,
+                worker_policy=worker_policy,
                 retry_store_path=_resolve_retry_store_path(retry_store_path, config),
                 max_jobs=max_jobs,
                 forever=forever,
@@ -381,6 +599,8 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 idle_sleep=idle_sleep,
                 release_delay=release_delay,
                 dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+                log_events=log_events,
+                mode="process",
             )
         if exit_code:
             raise typer.Exit(exit_code)
@@ -403,16 +623,39 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         timeout: float | None = typer.Option(None, "--timeout", min=0.0),
         idle_sleep: float = typer.Option(1.0, "--idle-sleep", min=0.001),
         release_delay: float = typer.Option(0.0, "--release-delay", min=0.0),
+        min_interval: float = typer.Option(0.0, "--min-interval", min=0.0),
+        circuit_breaker_failures: int = typer.Option(
+            0, "--circuit-breaker-failures", min=0
+        ),
+        circuit_breaker_cooldown: float = typer.Option(
+            0.0, "--circuit-breaker-cooldown", min=0.0
+        ),
         dead_letter_on_exhaustion: bool = typer.Option(
             True,
             "--dead-letter-on-exhaustion/--release-on-exhaustion",
         ),
+        log_events: bool = typer.Option(False, "--log-events"),
     ) -> None:
+        try:
+            _validate_worker_loop_options(
+                max_jobs=max_jobs,
+                forever=forever,
+            )
+        except ValueError as exc:
+            err_console.print(f"[red]{exc}[/red]")
+            raise typer.Exit(1) from exc
         persistent_queue = _queue(
             queue,
             _resolve_store_path(store_path, config),
             lease_timeout=lease_timeout,
         )
+        if worker_id is not None:
+            persistent_queue.record_worker_heartbeat(worker_id)
+        worker_policy = PersistentWorkerConfig(
+            min_interval=min_interval,
+            circuit_breaker_failures=circuit_breaker_failures,
+            circuit_breaker_cooldown=circuit_breaker_cooldown,
+        )
         with _shutdown_state() as shutdown:
             exit_code = _process_queue_messages(
                 persistent_queue,
@@ -420,6 +663,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 console=console,
                 err_console=err_console,
                 shutdown=shutdown,
+                worker_policy=worker_policy,
                 retry_store_path=_resolve_retry_store_path(retry_store_path, config),
                 max_jobs=max_jobs,
                 forever=forever,
@@ -430,6 +674,8 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 idle_sleep=idle_sleep,
                 release_delay=release_delay,
                 dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+                log_events=log_events,
+                mode="exec",
             )
         if exit_code:
             raise typer.Exit(exit_code)
@@ -453,6 +699,7 @@ def _complete_message(
     action: str,
     *,
     delay: float = 0.0,
+    log_events: bool = False,
 ) -> None:
     persistent_queue = _queue(queue, store_path)
     message = QueueMessage(id=message_id, value=None, queue=queue)
@@ -468,6 +715,14 @@ def _complete_message(
     if not completed:
         err_console.print(f"[red]message not found:[/red] {message_id}")
         raise typer.Exit(1)
+    if log_events:
+        _emit_event(
+            err_console,
+            f"queue.{action.replace('-', '_')}",
+            queue=queue,
+            message_id=message_id,
+            delay=delay if action == "release" else None,
+        )
     _print_json(console, {"id": message_id, "state": action})
 
 
@@ -481,6 +736,9 @@ def _process_message(
     max_tries: int,
     release_delay: float,
     dead_letter_on_exhaustion: bool,
+    log_events: bool = False,
+    mode: str = "process",
+    err_console: Any | None = None,
 ) -> _ProcessResult:
     retry_kwargs: dict[str, Any] = {
         "key": message.id,
@@ -507,24 +765,69 @@ def _process_message(
                 dead_letter_on_exhaustion=dead_letter_on_exhaustion,
                 error=exc,
             )
-            return _ProcessResult(processed=False, last_error=last_error)
+            if log_events and err_console is not None:
+                _emit_event(
+                    err_console,
+                    f"{mode}.dead_letter",
+                    queue=message.queue,
+                    message_id=message.id,
+                    attempts=message.attempts,
+                    leased_by=message.leased_by,
+                    last_error=last_error,
+                )
+            return _ProcessResult(
+                processed=False,
+                last_error=last_error,
+                final_state="dead-letter",
+                permanent_failure=False,
+            )
         except Exception as exc:
             last_error = _error_payload(exc)
             record = retryer.get_record(message.id)
             exhausted = record is not None and record.exhausted
-            if exhausted:
+            permanent_failure = is_permanent_failure(exc)
+            if permanent_failure or exhausted:
                 _ = _finish_failed_message(
                     queue,
                     message,
                     release_delay=release_delay,
-                    dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+                    dead_letter_on_exhaustion=True
+                    if permanent_failure
+                    else dead_letter_on_exhaustion,
                     error=exc,
                 )
+                final_state = "dead-letter"
             else:
                 _ = queue.release(message, delay=release_delay, error=exc)
-            return _ProcessResult(processed=False, last_error=last_error)
+                final_state = "release"
+            if log_events and err_console is not None:
+                _emit_event(
+                    err_console,
+                    f"{mode}.{final_state.replace('-', '_')}",
+                    queue=message.queue,
+                    message_id=message.id,
+                    attempts=message.attempts,
+                    leased_by=message.leased_by,
+                    last_error=last_error,
+                )
+            return _ProcessResult(
+                processed=False,
+                last_error=last_error,
+                final_state=final_state,
+                permanent_failure=permanent_failure,
+            )
 
-        return _ProcessResult(processed=queue.ack(message))
+        acked = queue.ack(message)
+        if log_events and err_console is not None:
+            _emit_event(
+                err_console,
+                f"{mode}.ack",
+                queue=message.queue,
+                message_id=message.id,
+                attempts=message.attempts,
+                leased_by=message.leased_by,
+            )
+        return _ProcessResult(processed=acked, final_state="acked")
     finally:
         if owned_retry_store is not None:
             owned_retry_store.close()
@@ -537,6 +840,7 @@ def _process_queue_messages(
     console: Any,
     err_console: Any,
     shutdown: _ShutdownState,
+    worker_policy: PersistentWorkerConfig | None = None,
     retry_store_path: str | None,
     max_jobs: int,
     forever: bool,
@@ -547,8 +851,14 @@ def _process_queue_messages(
     idle_sleep: float,
     release_delay: float,
     dead_letter_on_exhaustion: bool,
+    log_events: bool = False,
+    mode: str = "process",
 ) -> int:
     processed = 0
+    policy_state = WorkerPolicyState()
+    resolved_policy = (
+        worker_policy if worker_policy is not None else PersistentWorkerConfig()
+    )
 
     while forever or processed < max_jobs:
         if shutdown.requested:
@@ -556,6 +866,10 @@ def _process_queue_messages(
                 _print_json(console, {"state": "stopped", "processed": processed})
             return 0
 
+        if worker_id is not None:
+            queue.record_worker_heartbeat(worker_id)
+        _sleep_for_policy(policy_state, resolved_policy)
+
         try:
             message = queue.get_message(
                 block=block,
@@ -568,10 +882,22 @@ def _process_queue_messages(
                     time.sleep(idle_sleep)
                 continue
             if processed == 0:
-                err_console.print("[yellow]queue is empty[/yellow]")
+                if err_console is not None:
+                    err_console.print("[yellow]queue is empty[/yellow]")
                 return 1
             break
 
+        if log_events and err_console is not None:
+            _emit_event(
+                err_console,
+                f"{mode}.lease",
+                queue=message.queue,
+                message_id=message.id,
+                leased_by=message.leased_by,
+                attempts=message.attempts,
+                leased_until=message.leased_until,
+            )
+
         result = _process_message(
             queue,
             message,
@@ -580,16 +906,39 @@ def _process_queue_messages(
             max_tries=max_tries,
             release_delay=release_delay,
             dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+            log_events=log_events,
+            mode=mode,
+            err_console=err_console,
         )
         if result.processed:
             processed += 1
+            _record_success(policy_state)
+            if worker_id is not None:
+                queue.record_worker_heartbeat(worker_id)
             _print_json(console, {"id": message.id, "state": "acked"})
             continue
 
+        _record_failure(
+            policy_state,
+            resolved_policy,
+            permanent=result.permanent_failure,
+        )
+
+        payload = (
+            _message_payload(current_message)
+            if (current_message := queue.inspect(message.id)) is not None
+            else {"id": message.id, "queue": message.queue}
+        )
         _print_json(
             console,
-            {"id": message.id, "state": "failed", "last_error": result.last_error},
+            {
+                **payload,
+                "state": "failed",
+                "last_error": result.last_error,
+            },
         )
+        if worker_id is not None:
+            queue.record_worker_heartbeat(worker_id)
         if forever:
             continue
         return 1
@@ -605,10 +954,24 @@ def _print_dead_letters(
     watch: bool,
     interval: float,
     shutdown: _ShutdownState,
+    summary: bool,
+    min_attempts: int | None,
+    max_attempts: int | None,
+    error_contains: str | None,
+    failed_within: float | None,
 ) -> None:
     while True:
-        messages = queue.dead_letters(limit=limit)
-        _print_json(console, [_message_payload(message) for message in messages])
+        messages = _filter_dead_letters(
+            queue.dead_letters(limit=limit),
+            min_attempts=min_attempts,
+            max_attempts=max_attempts,
+            error_contains=error_contains,
+            failed_within=failed_within,
+        )
+        if summary:
+            _print_json(console, _dead_letter_summary(messages))
+        else:
+            _print_json(console, [_message_payload(message) for message in messages])
         if not watch:
             return
         if shutdown.requested:
@@ -624,10 +987,17 @@ def _print_queue_stats(
     console: Any,
     watch: bool,
     interval: float,
+    stale_after: float | None,
     shutdown: _ShutdownState,
 ) -> None:
     while True:
-        _print_json(console, queue.stats().as_dict())
+        stats = queue.stats()
+        payload = stats.as_dict()
+        if stale_after is not None:
+            payload["worker_health"] = _worker_health_summary(
+                stats.last_seen_by_worker_id, stale_after=stale_after
+            )
+        _print_json(console, payload)
         if not watch:
             return
         if shutdown.requested:
@@ -643,6 +1013,11 @@ def _poll_timeout(forever: bool, block: bool, timeout: float | None) -> float |
     return timeout
 
 
+def _validate_worker_loop_options(*, max_jobs: int, forever: bool) -> None:
+    if forever and max_jobs != 1:
+        raise ValueError("pass either --forever or --max-jobs, not both")
+
+
 @contextmanager
 def _shutdown_state() -> Iterator[_ShutdownState]:
     state = _ShutdownState()
@@ -757,6 +1132,40 @@ def _resolve_retry_store_path(explicit: str | None, config: dict[str, Any]) -> s
     return str(value)
 
 
+def _resolve_dead_letter_ttl(
+    explicit: float | None, config: dict[str, Any]
+) -> float | None:
+    value = explicit if explicit is not None else config.get("dead_letter_ttl_seconds")
+    if value is None:
+        return None
+    return float(value)
+
+
+def _resolve_retry_record_ttl(
+    explicit: float | None, config: dict[str, Any]
+) -> float | None:
+    value = explicit if explicit is not None else config.get("retry_record_ttl_seconds")
+    if value is None:
+        return None
+    return float(value)
+
+
+def _retention_settings(config: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "dead_letter_ttl_seconds": config.get("dead_letter_ttl_seconds"),
+        "retry_record_ttl_seconds": config.get("retry_record_ttl_seconds"),
+    }
+
+
+def _coerce_config_value(key: str, value: str) -> str | float:
+    if key in {"dead_letter_ttl_seconds", "retry_record_ttl_seconds"}:
+        ttl = float(value)
+        if ttl < 0:
+            raise ValueError(f"{key} must be greater than or equal to zero")
+        return ttl
+    return value
+
+
 def _load_callable(spec: str) -> Callable[[Any], Any]:
     module_name, separator, attribute = spec.partition(":")
     if not separator or not module_name or not attribute:
@@ -813,6 +1222,8 @@ def _message_payload(message: QueueMessage) -> dict[str, Any]:
         "leased_by": message.leased_by,
         "last_error": message.last_error,
         "failed_at": message.failed_at,
+        "dedupe_key": message.dedupe_key,
+        "attempt_history": message.attempt_history,
     }
 
 
@@ -820,6 +1231,108 @@ def _json_dumps(value: Any) -> str:
     return json.dumps(value, ensure_ascii=False, sort_keys=True)
 
 
+def _filter_dead_letters(
+    messages: list[QueueMessage],
+    *,
+    min_attempts: int | None,
+    max_attempts: int | None,
+    error_contains: str | None,
+    failed_within: float | None,
+) -> list[QueueMessage]:
+    now = time.time()
+    error_filter = error_contains.lower() if error_contains is not None else None
+    cutoff = None if failed_within is None else now - failed_within
+    filtered: list[QueueMessage] = []
+    for message in messages:
+        if min_attempts is not None and message.attempts < min_attempts:
+            continue
+        if max_attempts is not None and message.attempts > max_attempts:
+            continue
+        if cutoff is not None:
+            failed_at = message.failed_at
+            if failed_at is None or failed_at < cutoff:
+                continue
+        if error_filter is not None:
+            last_error = message.last_error or {}
+            haystack = " ".join(
+                str(part)
+                for part in (
+                    last_error.get("type"),
+                    last_error.get("message"),
+                    last_error.get("command"),
+                    last_error.get("stderr"),
+                )
+                if part is not None
+            ).lower()
+            if error_filter not in haystack:
+                continue
+        filtered.append(message)
+    return filtered
+
+
+def _dead_letter_summary(messages: list[QueueMessage]) -> dict[str, Any]:
+    by_error_type = Counter[str]()
+    by_attempts = Counter[str]()
+    by_worker_id = Counter[str]()
+    failed_ats: list[float] = []
+    for message in messages:
+        error_type = "None"
+        if message.last_error is not None:
+            error_type = str(message.last_error.get("type") or "None")
+        by_error_type[error_type] += 1
+        by_attempts[str(message.attempts)] += 1
+        worker_id = _last_attempt_worker_id(message)
+        if worker_id is not None:
+            by_worker_id[worker_id] += 1
+        if message.failed_at is not None:
+            failed_ats.append(message.failed_at)
+    summary: dict[str, Any] = {
+        "count": len(messages),
+        "by_error_type": dict(sorted(by_error_type.items())),
+        "by_attempts": dict(sorted(by_attempts.items(), key=lambda item: int(item[0]))),
+        "by_worker_id": dict(sorted(by_worker_id.items())),
+    }
+    if failed_ats:
+        summary["failed_at"] = {
+            "oldest": min(failed_ats),
+            "newest": max(failed_ats),
+        }
+    return summary
+
+
+def _worker_health_summary(
+    last_seen_by_worker_id: dict[str, float], *, stale_after: float
+) -> dict[str, Any]:
+    now = time.time()
+    workers_active: dict[str, float] = {}
+    workers_stale: dict[str, float] = {}
+    for worker_id, last_seen in sorted(last_seen_by_worker_id.items()):
+        age = max(now - last_seen, 0.0)
+        if age > stale_after:
+            workers_stale[worker_id] = age
+        else:
+            workers_active[worker_id] = age
+    return {
+        "stale_after_seconds": stale_after,
+        "active": {
+            "count": len(workers_active),
+            "by_worker_id": workers_active,
+        },
+        "stale": {
+            "count": len(workers_stale),
+            "by_worker_id": workers_stale,
+        },
+    }
+
+
+def _last_attempt_worker_id(message: QueueMessage) -> str | None:
+    for event in reversed(message.attempt_history):
+        if event.get("type") == "leased":
+            worker_id = event.get("leased_by")
+            return None if worker_id is None else str(worker_id)
+    return None
+
+
 def _format_command(command: list[str]) -> str:
     return " ".join(command)
 
@@ -832,3 +1345,11 @@ def _truncate_output(value: str, *, limit: int = 500) -> str:
 
 def _print_json(console: Any, value: Any) -> None:
     console.print_json(_json_dumps(value))
+
+
+def _emit_event(console: Any, event: str, **fields: Any) -> None:
+    payload = {
+        "event": event,
+        **{key: value for key, value in fields.items() if value is not None},
+    }
+    _print_json(console, payload)