localqueue 0.1.0__tar.gz → 0.2.0__tar.gz

{localqueue-0.1.0 → localqueue-0.2.0}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## Unreleased
+
+## 0.2.0
+
+- Add `queue stats --watch` for monitoring queue counts while workers run.
+- Add `queue dead --watch` for repeatedly listing dead letters.
+- Add `queue requeue-dead --all` for bulk recovery after a fix.
+- Add structured `command not found` handling for `queue exec`.
+- Add `examples/process_webhook.sh` as a shell/curl worker example.
+
+## 0.1.1
+
+- Add project URLs for PyPI metadata.
+- Document `queue exec` command-failure fields stored in `last_error`.
+
 ## 0.1.0
 
 - Add persistent retry wrappers for sync and async Tenacity retryers.

{localqueue-0.1.0 → localqueue-0.2.0}/PKG-INFO

@@ -1,7 +1,11 @@
 Metadata-Version: 2.4
 Name: localqueue
-Version: 0.1.0
+Version: 0.2.0
 Summary: Durable local queues for Python, with persistent retry state powered by Tenacity.
+Project-URL: Homepage, https://brunoportis.github.io/localqueue/
+Project-URL: Documentation, https://brunoportis.github.io/localqueue/
+Project-URL: Repository, https://github.com/brunoportis/localqueue
+Project-URL: Issues, https://github.com/brunoportis/localqueue/issues
 Author: Bruno Portis
 License-Expression: MIT
 License-File: LICENSE
@@ -121,9 +125,12 @@ localqueue queue add emails --value '{"to":"user@example.com"}'
 echo '{"to":"user@example.com"}' | localqueue queue add emails
 localqueue queue size emails
 localqueue queue stats emails
+localqueue queue stats emails --watch --interval 1
 localqueue queue inspect emails <message-id>
 localqueue queue dead emails
+localqueue queue dead emails --watch --interval 1
 localqueue queue requeue-dead emails <message-id>
+localqueue queue requeue-dead emails --all
 localqueue queue pop emails --worker-id worker-1
 localqueue queue ack emails <message-id>
 localqueue queue exec emails -- python scripts/send_email.py
@@ -157,8 +164,13 @@ retry, release, and dead-letter rules as `queue process`.
 localqueue queue exec emails -- python scripts/send_email.py
 localqueue queue exec webhooks -- curl -X POST https://example.com/hook -d @-
 localqueue queue exec emails -- sh -c 'jq -r .to | xargs -I{} curl https://example.com/{}'
+localqueue queue exec webhooks -- sh examples/process_webhook.sh
 ```
 
+Command output is captured so the CLI can keep printing its own JSON status.
+When a command fails, `last_error` includes the command, exit code, stdout, and
+stderr, truncated for inspection.
+
 Use `--forever` for a long-running worker. When interrupted with `SIGINT` or
 `SIGTERM`, the CLI finishes the current message before stopping.
 
@@ -276,6 +288,10 @@ Use `queue stats` when you need ready, delayed, inflight, dead-letter, and total
 counts instead of only ready messages. Use `queue inspect` to inspect one
 message by id, `queue dead` to list dead-letter messages, and
 `queue requeue-dead` to retry one after the failure cause is fixed.
+Use `queue stats --watch` to print those counts repeatedly while local workers
+are running.
+Use `queue dead --watch` to keep an eye on newly failed jobs, and
+`queue requeue-dead --all` once you have fixed the underlying issue.
 
 ## Operational boundaries
 

{localqueue-0.1.0 → localqueue-0.2.0}/README.md

@@ -90,9 +90,12 @@ localqueue queue add emails --value '{"to":"user@example.com"}'
 echo '{"to":"user@example.com"}' | localqueue queue add emails
 localqueue queue size emails
 localqueue queue stats emails
+localqueue queue stats emails --watch --interval 1
 localqueue queue inspect emails <message-id>
 localqueue queue dead emails
+localqueue queue dead emails --watch --interval 1
 localqueue queue requeue-dead emails <message-id>
+localqueue queue requeue-dead emails --all
 localqueue queue pop emails --worker-id worker-1
 localqueue queue ack emails <message-id>
 localqueue queue exec emails -- python scripts/send_email.py
@@ -126,8 +129,13 @@ retry, release, and dead-letter rules as `queue process`.
 localqueue queue exec emails -- python scripts/send_email.py
 localqueue queue exec webhooks -- curl -X POST https://example.com/hook -d @-
 localqueue queue exec emails -- sh -c 'jq -r .to | xargs -I{} curl https://example.com/{}'
+localqueue queue exec webhooks -- sh examples/process_webhook.sh
 ```
 
+Command output is captured so the CLI can keep printing its own JSON status.
+When a command fails, `last_error` includes the command, exit code, stdout, and
+stderr, truncated for inspection.
+
 Use `--forever` for a long-running worker. When interrupted with `SIGINT` or
 `SIGTERM`, the CLI finishes the current message before stopping.
 
@@ -245,6 +253,10 @@ Use `queue stats` when you need ready, delayed, inflight, dead-letter, and total
 counts instead of only ready messages. Use `queue inspect` to inspect one
 message by id, `queue dead` to list dead-letter messages, and
 `queue requeue-dead` to retry one after the failure cause is fixed.
+Use `queue stats --watch` to print those counts repeatedly while local workers
+are running.
+Use `queue dead --watch` to keep an eye on newly failed jobs, and
+`queue requeue-dead --all` once you have fixed the underlying issue.
 
 ## Operational boundaries
 

{localqueue-0.1.0 → localqueue-0.2.0}/docs/api.md

@@ -94,6 +94,10 @@ Dataclass returned by `put()` and `get_message()`.
 | `last_error` | structured error from the most recent failed processing attempt, if recorded |
 | `failed_at` | timestamp for `last_error`, if recorded |
 
+For `localqueue queue exec` failures, `last_error` also includes `command`,
+`exit_code`, `stdout`, and `stderr` fields so command workers can be inspected
+from `queue inspect` and `queue dead`.
+
 #### `QueueStats`
 
 Dataclass returned by `stats()`.

{localqueue-0.1.0 → localqueue-0.2.0}/docs/index.md

@@ -50,6 +50,8 @@ localqueue queue exec emails -- python scripts/send_email.py
 `queue exec` writes the message value to the command's stdin as JSON. Exit code
 `0` acknowledges the message. Any other exit code is treated as a failed handler
 attempt and follows the configured retry, release, and dead-letter policy.
+Command failures are recorded in `last_error` with the command, exit code,
+stdout, and stderr.
 
 Run a continuous local worker with `--forever`. `SIGINT` and `SIGTERM` request a
 graceful stop after the current message finishes.
@@ -62,6 +64,27 @@ localqueue queue process emails myapp.workers:send_email \
   --max-tries 5
 ```
 
+Watch queue counts while workers run:
+
+```bash
+localqueue queue stats emails --watch --interval 1
+```
+
+For shell workers, `examples/process_webhook.sh` shows the same pattern with
+`jq` and `curl`.
+
+Inspect dead letters while workers are running:
+
+```bash
+localqueue queue dead emails --watch --interval 2
+```
+
+After fixing a problem, requeue all dead letters with:
+
+```bash
+localqueue queue requeue-dead emails --all
+```
+
 For a local smoke test, enqueue one job and process it with the bundled example
 handler:
 

{localqueue-0.1.0 → localqueue-0.2.0}/docs/queues.md

@@ -91,16 +91,29 @@ external command instead of an importable Python function.
 localqueue queue exec jobs -- python scripts/process_job.py
 localqueue queue exec webhooks -- curl -X POST https://example.com/hook -d @-
 localqueue queue exec jobs -- sh -c 'jq -r .id | xargs -I{} ./process-job {}'
+localqueue queue exec webhooks -- sh examples/process_webhook.sh
 ```
 
 The command receives `message.value` on stdin as JSON. `localqueue` captures the
 command output so the CLI can keep printing its own JSON status. Exit code `0`
-acks the message. Any other exit code raises a command failure, records the exit
-code and stderr in `last_error`, and applies the same retry, release, or
-dead-letter behavior as `queue process`.
+acks the message. Any other exit code raises a command failure and applies the
+same retry, release, or dead-letter behavior as `queue process`.
+
+Command failures are stored in `last_error` with structured fields:
+
+| Field | Meaning |
+| --- | --- |
+| `type` | `_CommandExecutionError` |
+| `message` | human-readable command failure |
+| `command` | argv list that was executed |
+| `exit_code` | command exit code |
+| `stdout` | captured stdout, truncated for inspection |
+| `stderr` | captured stderr, truncated for inspection |
 
 Commands are executed without an implicit shell. Use `sh -c` explicitly when you
 want pipes, redirects, shell expansion, or multiple commands.
+See `examples/process_webhook.sh` for a shell worker that reads JSON, extracts
+fields with `jq`, and posts them with `curl`.
 
 ## Queue-style usage
 
@@ -305,3 +318,26 @@ by id without changing state. `dead_letters()` lists dead-letter messages for
 inspection. `requeue_dead()` moves a dead-letter message back to the ready queue.
 `purge()` removes all records for that queue, including ready, inflight, and
 dead-letter records.
+
+From the CLI, use `queue stats --watch` to monitor those counts while workers
+are running:
+
+```bash
+localqueue queue stats jobs --watch --interval 1
+```
+
+Each sample is printed as JSON with `ready`, `delayed`, `inflight`, `dead`, and
+`total` counts. Stop the watch with `Ctrl-C`.
+
+Use `queue dead --watch` to keep an eye on recent failures:
+
+```bash
+localqueue queue dead jobs --watch --interval 2
+```
+
+When the underlying problem is fixed, `queue requeue-dead --all` moves every
+dead letter back to ready delivery:
+
+```bash
+localqueue queue requeue-dead jobs --all
+```

{localqueue-0.1.0 → localqueue-0.2.0}/docs/release.md

@@ -104,4 +104,4 @@ Before `1.0.0`, use this convention:
 
 ## Current Release
 
-The current planned release is `0.1.0`.
+The current planned release is `0.2.0`.

localqueue-0.2.0/examples/process_webhook.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+
+payload="$(cat)"
+url="$(printf '%s' "$payload" | jq -r '.url')"
+body="$(printf '%s' "$payload" | jq -c '.payload')"
+
+curl -fsS -X POST "$url" \
+  -H 'content-type: application/json' \
+  -d "$body"

{localqueue-0.1.0 → localqueue-0.2.0}/localqueue/cli.py

@@ -67,6 +67,17 @@ class _CommandExecutionError(RuntimeError):
         )
 
 
+class _CommandNotFoundError(_CommandExecutionError):
+    def __init__(self, command: list[str]) -> None:
+        super().__init__(
+            command,
+            exit_code=127,
+            stdout="",
+            stderr="command not found",
+        )
+        self.args = (f"command not found: {_format_command(command)}",)
+
+
 def main(args: list[str] | None = None) -> None:
     try:
         from rich.console import Console
@@ -251,29 +262,65 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     def queue_stats(
         queue: str,
         store_path: str | None = typer.Option(None, "--store-path"),
+        watch: bool = typer.Option(False, "--watch"),
+        interval: float = typer.Option(1.0, "--interval", min=0.001),
     ) -> None:
-        stats = _queue(queue, _resolve_store_path(store_path, config)).stats()
-        _print_json(console, stats.as_dict())
+        persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
+        with _shutdown_state() as shutdown:
+            _print_queue_stats(
+                persistent_queue,
+                console=console,
+                watch=watch,
+                interval=interval,
+                shutdown=shutdown,
+            )
 
     @queue_app.command("dead")
     def queue_dead(
         queue: str,
         store_path: str | None = typer.Option(None, "--store-path"),
         limit: int | None = typer.Option(None, "--limit", min=0),
+        watch: bool = typer.Option(False, "--watch"),
+        interval: float = typer.Option(1.0, "--interval", min=0.001),
     ) -> None:
-        messages = _queue(queue, _resolve_store_path(store_path, config)).dead_letters(
-            limit=limit
-        )
-        _print_json(console, [_message_payload(message) for message in messages])
+        persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
+        with _shutdown_state() as shutdown:
+            _print_dead_letters(
+                persistent_queue,
+                console=console,
+                limit=limit,
+                watch=watch,
+                interval=interval,
+                shutdown=shutdown,
+            )
 
     @queue_app.command("requeue-dead")
     def queue_requeue_dead(
         queue: str,
-        message_id: str,
+        message_id: str | None = typer.Argument(None),
         store_path: str | None = typer.Option(None, "--store-path"),
         delay: float = typer.Option(0.0, "--delay", min=0.0),
+        all_: bool = typer.Option(False, "--all"),
     ) -> None:
         persistent_queue = _queue(queue, _resolve_store_path(store_path, config))
+        if all_:
+            if message_id is not None:
+                err_console.print(
+                    "[red]pass either a message id or --all, not both[/red]"
+                )
+                raise typer.Exit(1)
+            messages = persistent_queue.dead_letters()
+            requeued = 0
+            for message in messages:
+                if persistent_queue.requeue_dead(message, delay=delay):
+                    requeued += 1
+            _print_json(console, {"requeued": requeued})
+            return
+        if message_id is None:
+            err_console.print(
+                "[red]pass a message id or use --all to requeue dead letters[/red]"
+            )
+            raise typer.Exit(1)
         message = QueueMessage(id=message_id, value=None, queue=queue)
         if not persistent_queue.requeue_dead(message, delay=delay):
             err_console.print(f"[red]dead-letter message not found:[/red] {message_id}")
@@ -550,6 +597,46 @@ def _process_queue_messages(
     return 0
 
 
+def _print_dead_letters(
+    queue: PersistentQueue,
+    *,
+    console: Any,
+    limit: int | None,
+    watch: bool,
+    interval: float,
+    shutdown: _ShutdownState,
+) -> None:
+    while True:
+        messages = queue.dead_letters(limit=limit)
+        _print_json(console, [_message_payload(message) for message in messages])
+        if not watch:
+            return
+        if shutdown.requested:
+            return
+        time.sleep(interval)
+        if shutdown.requested:
+            return
+
+
+def _print_queue_stats(
+    queue: PersistentQueue,
+    *,
+    console: Any,
+    watch: bool,
+    interval: float,
+    shutdown: _ShutdownState,
+) -> None:
+    while True:
+        _print_json(console, queue.stats().as_dict())
+        if not watch:
+            return
+        if shutdown.requested:
+            return
+        time.sleep(interval)
+        if shutdown.requested:
+            return
+
+
 def _poll_timeout(forever: bool, block: bool, timeout: float | None) -> float | None:
     if forever and block and timeout is None:
         return 0.5
@@ -692,13 +779,16 @@ def _command_handler(command: list[str]) -> Callable[[Any], None]:
         raise ValueError("command cannot be empty")
 
     def run_command(value: Any) -> None:
-        completed = subprocess.run(
-            command,
-            input=_json_dumps(value) + "\n",
-            capture_output=True,
-            text=True,
-            check=False,
-        )
+        try:
+            completed = subprocess.run(
+                command,
+                input=_json_dumps(value) + "\n",
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+        except FileNotFoundError as exc:
+            raise _CommandNotFoundError(command) from exc
         if completed.returncode != 0:
             raise _CommandExecutionError(
                 command,

{localqueue-0.1.0 → localqueue-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "localqueue"
-version = "0.1.0"
+version = "0.2.0"
 description = "Durable local queues for Python, with persistent retry state powered by Tenacity."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -43,6 +43,12 @@ all = [
 [project.scripts]
 localqueue = "localqueue.cli:main"
 
+[project.urls]
+Homepage = "https://brunoportis.github.io/localqueue/"
+Documentation = "https://brunoportis.github.io/localqueue/"
+Repository = "https://github.com/brunoportis/localqueue"
+Issues = "https://github.com/brunoportis/localqueue/issues"
+
 [build-system]
 requires = ["hatchling>=1.27.0"]
 build-backend = "hatchling.build"

{localqueue-0.1.0 → localqueue-0.2.0}/tests/test_cli.py

@@ -28,6 +28,8 @@ from localqueue.cli import (
     _load_callable,
     _load_config,
     _parse_json,
+    _print_dead_letters,
+    _print_queue_stats,
     _process_message,
     _process_queue_messages,
     _read_value,
@@ -380,6 +382,30 @@ class CliTests(unittest.TestCase):
             self.assertEqual(missing_requeue.exit_code, 1)
             self.assertIn("dead-letter message not found", missing_requeue.output)
 
+    def test_print_dead_letters_watch_stops_on_shutdown(self) -> None:
+        queue = PersistentQueue("emails", store=MemoryQueueStore())
+        _ = queue.put({"to": "user@example.com"})
+        message = queue.get_message()
+        self.assertTrue(queue.dead_letter(message, error=RuntimeError("bad")))
+        console = _JsonConsole()
+        shutdown = _ShutdownState()
+
+        def request_shutdown(_interval: float) -> None:
+            shutdown.requested = True
+
+        with mock.patch("time.sleep", side_effect=request_shutdown):
+            _print_dead_letters(
+                queue,
+                console=console,
+                limit=None,
+                watch=True,
+                interval=0.001,
+                shutdown=shutdown,
+            )
+
+        self.assertEqual(len(console.values), 1)
+        self.assertEqual(console.values[0][0]["id"], message.id)
+
     def test_queue_process_command_acks_successful_handler(self) -> None:
         with tempfile.TemporaryDirectory() as tmpdir:
             tmp_path = Path(tmpdir)
@@ -496,6 +522,40 @@ class CliTests(unittest.TestCase):
             self.assertEqual(result.exit_code, 1)
             self.assertIn("module:function", result.output)
 
+    def test_queue_exec_reports_missing_command(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            store_path = str(Path(tmpdir) / "queues")
+            retry_store_path = str(Path(tmpdir) / "retries.sqlite3")
+            queue = PersistentQueue("emails", store_path=store_path)
+            message = queue.put({"to": "user@example.com"})
+
+            result = self._invoke(
+                [
+                    "queue",
+                    "exec",
+                    "emails",
+                    "--store-path",
+                    store_path,
+                    "--retry-store-path",
+                    retry_store_path,
+                    "--max-tries",
+                    "1",
+                    "--",
+                    "definitely-not-a-real-command-123",
+                ]
+            )
+
+            self.assertEqual(result.exit_code, 1)
+            payload = json.loads(result.stdout)
+            self.assertEqual(payload["id"], message.id)
+            self.assertEqual(payload["state"], "failed")
+            self.assertEqual(payload["last_error"]["type"], "_CommandNotFoundError")
+            self.assertEqual(payload["last_error"]["exit_code"], 127)
+            self.assertEqual(
+                payload["last_error"]["command"], ["definitely-not-a-real-command-123"]
+            )
+            self.assertIn("command not found", payload["last_error"]["message"])
+
     def test_command_handler_passes_message_value_as_json_stdin(self) -> None:
         with tempfile.TemporaryDirectory() as tmpdir:
             tmp_path = Path(tmpdir)
@@ -610,6 +670,48 @@ class CliTests(unittest.TestCase):
             self.assertEqual(dead_letters[0].last_error["exit_code"], 7)
             self.assertEqual(dead_letters[0].last_error["stderr"], "cannot deliver")
 
+    def test_queue_requeue_dead_all_moves_every_dead_message(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            store_path = str(Path(tmpdir) / "queues")
+            persistent_queue = PersistentQueue("emails", store_path=store_path)
+            first = persistent_queue.put({"to": "one@example.com"})
+            second = persistent_queue.put({"to": "two@example.com"})
+            self.assertTrue(
+                persistent_queue.dead_letter(
+                    persistent_queue.get_message(), error=RuntimeError("bad 1")
+                )
+            )
+            self.assertTrue(
+                persistent_queue.dead_letter(
+                    persistent_queue.get_message(), error=RuntimeError("bad 2")
+                )
+            )
+
+            result = self._invoke(
+                [
+                    "queue",
+                    "requeue-dead",
+                    "emails",
+                    "--all",
+                    "--store-path",
+                    store_path,
+                ]
+            )
+
+            self.assertEqual(result.exit_code, 0, result.output)
+            self.assertEqual(json.loads(result.stdout), {"requeued": 2})
+            fresh_queue = PersistentQueue("emails", store_path=store_path)
+            self.assertEqual(fresh_queue.dead_letters(), [])
+            self.assertEqual(fresh_queue.qsize(), 2)
+            first_message = fresh_queue.inspect(first.id)
+            second_message = fresh_queue.inspect(second.id)
+            self.assertIsNotNone(first_message)
+            self.assertIsNotNone(second_message)
+            assert first_message is not None
+            assert second_message is not None
+            self.assertEqual(first_message.state, "ready")
+            self.assertEqual(second_message.state, "ready")
+
     def test_process_message_acks_successful_handler(self) -> None:
         queue = PersistentQueue("emails", store=MemoryQueueStore())
         _ = queue.put({"to": "user@example.com"})
@@ -724,6 +826,29 @@ class CliTests(unittest.TestCase):
         self.assertEqual(console.values, [])
         self.assertEqual(err_console.messages, ["[yellow]queue is empty[/yellow]"])
 
+    def test_print_queue_stats_watch_stops_on_shutdown(self) -> None:
+        queue = PersistentQueue("emails", store=MemoryQueueStore())
+        _ = queue.put({"to": "user@example.com"})
+        console = _JsonConsole()
+        shutdown = _ShutdownState()
+
+        def request_shutdown(_interval: float) -> None:
+            shutdown.requested = True
+
+        with mock.patch("time.sleep", side_effect=request_shutdown):
+            _print_queue_stats(
+                queue,
+                console=console,
+                watch=True,
+                interval=0.001,
+                shutdown=shutdown,
+            )
+
+        self.assertEqual(
+            console.values,
+            [{"ready": 1, "delayed": 0, "inflight": 0, "dead": 0, "total": 1}],
+        )
+
     def test_process_queue_messages_forever_stops_after_current_message(self) -> None:
         queue = PersistentQueue("emails", store=MemoryQueueStore())
         _ = queue.put({"to": "first@example.com"})

{localqueue-0.1.0 → localqueue-0.2.0}/uv.lock

@@ -200,7 +200,7 @@ wheels = [
 
 [[package]]
 name = "localqueue"
-version = "0.1.0"
+version = "0.2.0"
 source = { editable = "." }
 dependencies = [
     { name = "tenacity" },