localqueue 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

localqueue/cli.py

@@ -8,13 +8,11 @@ import subprocess
 import sys
 import time
 from collections import Counter
-from collections.abc import Callable
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
 from queue import Empty
-from types import FrameType
-from typing import Any, Iterator
+from typing import Any, Iterator, TYPE_CHECKING
 
 from tenacity import wait_none
 
@@ -35,6 +33,10 @@ from .worker import (
     _sleep_for_policy,
 )
 
+if TYPE_CHECKING:
+    from types import FrameType
+    from collections.abc import Callable
+
 DEFAULT_STORE_PATH = str(default_queue_store_path())
 DEFAULT_RETRY_STORE_PATH = str(default_retry_store_path())
 CONFIG_FILENAME = "config.yaml"
@@ -114,7 +116,20 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     app.add_typer(queue_app, name="queue")
     app.add_typer(retry_app, name="retry")
     app.add_typer(config_app, name="config")
+    _register_config_commands(config_app, typer, yaml, console, err_console, config)
+    _register_queue_commands(queue_app, typer, console, err_console, config)
+    _register_retry_commands(retry_app, typer, console, err_console, config)
+    return app
 
+
+def _register_config_commands(
+    config_app: Any,
+    typer: Any,
+    yaml: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @config_app.command("path")
     def config_path() -> None:
         console.print(str(_config_path()))
@@ -181,6 +196,26 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         config.update(updated)
         _print_json(console, config)
 
+
+def _register_queue_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
+    _register_queue_basic_commands(queue_app, typer, console, err_console, config)
+    _register_queue_admin_commands(queue_app, typer, console, err_console, config)
+    _register_queue_worker_commands(queue_app, typer, console, err_console, config)
+
+
+def _register_queue_basic_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("add")
     def queue_add(
         queue: str,
@@ -210,6 +245,17 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             )
         _print_json(console, _message_payload(message))
 
+    _register_queue_message_commands(queue_app, typer, console, err_console, config)
+    _register_queue_maintenance_commands(queue_app, typer, console, err_console, config)
+
+
+def _register_queue_message_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("pop")
     def queue_pop(
         queue: str,
@@ -319,6 +365,27 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     ) -> None:
         console.print(_queue(queue, _resolve_store_path(store_path, config)).qsize())
 
+
+def _register_queue_maintenance_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
+    _register_queue_stats_commands(queue_app, typer, console, err_console, config)
+    _register_queue_dead_commands(queue_app, typer, console, err_console, config)
+    _register_queue_requeue_commands(queue_app, typer, console, err_console, config)
+    _register_queue_purge_commands(queue_app, typer, console, err_console, config)
+
+
+def _register_queue_stats_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("stats")
     def queue_stats(
         queue: str,
@@ -362,6 +429,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             },
         )
 
+
+def _register_queue_dead_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("dead")
     def queue_dead(
         queue: str,
@@ -434,39 +509,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 failed_within=failed_within,
             )
 
-    @retry_app.command("prune")
-    def retry_prune(
-        older_than: float | None = typer.Option(None, "--older-than", min=0.0),
-        retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
-        dry_run: bool = typer.Option(False, "--dry-run"),
-    ) -> None:
-        store = SQLiteAttemptStore(_resolve_retry_store_path(retry_store_path, config))
-        try:
-            resolved_older_than = _resolve_retry_record_ttl(older_than, config)
-            if resolved_older_than is None:
-                err_console.print(
-                    "[red]pass --older-than or configure retry_record_ttl_seconds[/red]"
-                )
-                raise typer.Exit(1)
-            now = time.time()
-            deleted = (
-                store.count_exhausted_older_than(
-                    older_than=resolved_older_than, now=now
-                )
-                if dry_run
-                else store.prune_exhausted(older_than=resolved_older_than, now=now)
-            )
-        finally:
-            store.close()
-        _print_json(
-            console,
-            {
-                "dry_run": dry_run,
-                "older_than": resolved_older_than,
-                "would_delete" if dry_run else "deleted": deleted,
-            },
-        )
 
+def _register_queue_requeue_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("requeue-dead")
     def queue_requeue_dead(
         queue: str,
@@ -518,6 +568,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             )
         _print_json(console, {"id": message_id, "state": "requeued"})
 
+
+def _register_queue_purge_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("purge")
     def queue_purge(
         queue: str,
@@ -529,6 +587,55 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             _emit_event(err_console, "queue.purge", queue=queue, deleted=deleted)
         console.print(deleted)
 
+
+def _register_retry_commands(
+    retry_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
+    @retry_app.command("prune")
+    def retry_prune(
+        older_than: float | None = typer.Option(None, "--older-than", min=0.0),
+        retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
+        dry_run: bool = typer.Option(False, "--dry-run"),
+    ) -> None:
+        store = SQLiteAttemptStore(_resolve_retry_store_path(retry_store_path, config))
+        try:
+            resolved_older_than = _resolve_retry_record_ttl(older_than, config)
+            if resolved_older_than is None:
+                err_console.print(
+                    "[red]pass --older-than or configure retry_record_ttl_seconds[/red]"
+                )
+                raise typer.Exit(1)
+            now = time.time()
+            deleted = (
+                store.count_exhausted_older_than(
+                    older_than=resolved_older_than, now=now
+                )
+                if dry_run
+                else store.prune_exhausted(older_than=resolved_older_than, now=now)
+            )
+        finally:
+            store.close()
+        _print_json(
+            console,
+            {
+                "dry_run": dry_run,
+                "older_than": resolved_older_than,
+                "would_delete" if dry_run else "deleted": deleted,
+            },
+        )
+
+
+def _register_queue_admin_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("process")
     def queue_process(
         queue: str,
@@ -606,6 +713,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         if exit_code:
             raise typer.Exit(exit_code)
 
+
+def _register_queue_worker_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("exec")
     def queue_exec(
         queue: str,
@@ -681,8 +796,6 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         if exit_code:
             raise typer.Exit(exit_code)
 
-    return app
-
 
 def _queue(
     name: str, store_path: str, *, lease_timeout: float = 30.0
@@ -869,86 +982,53 @@ def _process_queue_messages(
                     _print_json(console, {"state": "stopped", "processed": processed})
                 return 0
 
-            if worker_id is not None:
-                queue.record_worker_heartbeat(worker_id)
-            _sleep_for_policy(policy_state, resolved_policy)
-
-            try:
-                message = queue.get_message(
-                    block=block,
-                    timeout=_poll_timeout(forever, block, timeout),
-                    leased_by=worker_id,
-                )
-            except Empty:
-                if forever:
-                    if not block:
-                        time.sleep(idle_sleep)
-                    continue
-                if processed == 0:
-                    if err_console is not None:
-                        err_console.print("[yellow]queue is empty[/yellow]")
-                    return 1
-                break
-
-            if log_events and err_console is not None:
-                _emit_event(
-                    err_console,
-                    f"{mode}.lease",
-                    queue=message.queue,
-                    message_id=message.id,
-                    leased_by=message.leased_by,
-                    attempts=message.attempts,
-                    leased_until=message.leased_until,
-                )
-
-            if retry_store_path is not None and owned_retry_store is None:
-                owned_retry_store = SQLiteAttemptStore(retry_store_path)
-
-            result = _process_message(
+            iteration, owned_retry_store, empty_queue = _process_queue_iteration(
                 queue,
-                message,
                 handler,
-                retry_store=owned_retry_store,
-                retry_store_path=None
-                if owned_retry_store is not None
-                else retry_store_path,
+                console=console,
+                err_console=err_console,
+                policy_state=policy_state,
+                resolved_policy=resolved_policy,
+                retry_store_path=retry_store_path,
+                owned_retry_store=owned_retry_store,
                 max_tries=max_tries,
+                worker_id=worker_id,
+                block=block,
+                timeout=timeout,
+                idle_sleep=idle_sleep,
                 release_delay=release_delay,
                 dead_letter_on_exhaustion=dead_letter_on_exhaustion,
                 log_events=log_events,
                 mode=mode,
-                err_console=err_console,
+                forever=forever,
             )
-            if result.processed:
+            if empty_queue:
+                if processed == 0 and err_console is not None:
+                    err_console.print("[yellow]queue is empty[/yellow]")
+                if processed == 0:
+                    return 1
+                break
+            assert iteration is not None
+            if _handle_processed_queue_result(
+                queue,
+                iteration.message,
+                iteration.process_result,
+                console=console,
+                worker_id=worker_id,
+                policy_state=policy_state,
+            ):
                 processed += 1
-                _record_success(policy_state)
-                if worker_id is not None:
-                    queue.record_worker_heartbeat(worker_id)
-                _print_json(console, {"id": message.id, "state": "acked"})
                 continue
-
-            _record_failure(
-                policy_state,
-                resolved_policy,
-                permanent=result.permanent_failure,
-            )
-
-            payload = (
-                _message_payload(current_message)
-                if (current_message := queue.inspect(message.id)) is not None
-                else {"id": message.id, "queue": message.queue}
-            )
-            _print_json(
-                console,
-                {
-                    **payload,
-                    "state": "failed",
-                    "last_error": result.last_error,
-                },
-            )
-            if worker_id is not None:
-                queue.record_worker_heartbeat(worker_id)
-            if forever:
+            if _handle_failed_queue_result(
+                queue,
+                iteration.message,
+                iteration.process_result,
+                console=console,
+                worker_id=worker_id,
+                policy_state=policy_state,
+                resolved_policy=resolved_policy,
+                forever=forever,
+            ):
                 continue
             return 1
 
@@ -958,6 +1038,137 @@ def _process_queue_messages(
             owned_retry_store.close()
 
 
+@dataclass(slots=True)
+class _QueueIterationResult:
+    message: QueueMessage
+    process_result: _ProcessResult
+
+
+def _process_queue_iteration(
+    queue: PersistentQueue,
+    handler: Callable[[Any], Any],
+    *,
+    console: Any,
+    err_console: Any,
+    policy_state: WorkerPolicyState,
+    resolved_policy: PersistentWorkerConfig,
+    retry_store_path: str | None,
+    owned_retry_store: SQLiteAttemptStore | None,
+    max_tries: int,
+    worker_id: str | None,
+    block: bool,
+    timeout: float | None,
+    idle_sleep: float,
+    release_delay: float,
+    dead_letter_on_exhaustion: bool,
+    log_events: bool,
+    mode: str,
+    forever: bool,
+) -> tuple[_QueueIterationResult | None, SQLiteAttemptStore | None, bool]:
+    if worker_id is not None:
+        queue.record_worker_heartbeat(worker_id)
+    _sleep_for_policy(policy_state, resolved_policy)
+
+    try:
+        message = queue.get_message(
+            block=block,
+            timeout=_poll_timeout(forever, block, timeout),
+            leased_by=worker_id,
+        )
+    except Empty:
+        if forever:
+            if not block:
+                time.sleep(idle_sleep)
+            return None, owned_retry_store, True
+        return None, owned_retry_store, True
+
+    if log_events and err_console is not None:
+        _emit_event(
+            err_console,
+            f"{mode}.lease",
+            queue=message.queue,
+            message_id=message.id,
+            leased_by=message.leased_by,
+            attempts=message.attempts,
+            leased_until=message.leased_until,
+        )
+
+    if retry_store_path is not None and owned_retry_store is None:
+        owned_retry_store = SQLiteAttemptStore(retry_store_path)
+
+    result = _process_message(
+        queue,
+        message,
+        handler,
+        retry_store=owned_retry_store,
+        retry_store_path=None if owned_retry_store is not None else retry_store_path,
+        max_tries=max_tries,
+        release_delay=release_delay,
+        dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+        log_events=log_events,
+        mode=mode,
+        err_console=err_console,
+    )
+    return (
+        _QueueIterationResult(message=message, process_result=result),
+        owned_retry_store,
+        False,
+    )
+
+
+def _handle_processed_queue_result(
+    queue: PersistentQueue,
+    message: QueueMessage,
+    result: _ProcessResult,
+    *,
+    console: Any,
+    worker_id: str | None,
+    policy_state: WorkerPolicyState,
+) -> bool:
+    if not result.processed:
+        return False
+    _record_success(policy_state)
+    if worker_id is not None:
+        queue.record_worker_heartbeat(worker_id)
+    _print_json(console, {"id": message.id, "state": "acked"})
+    return True
+
+
+def _handle_failed_queue_result(
+    queue: PersistentQueue,
+    message: QueueMessage,
+    result: _ProcessResult,
+    *,
+    console: Any,
+    worker_id: str | None,
+    policy_state: WorkerPolicyState,
+    resolved_policy: PersistentWorkerConfig,
+    forever: bool,
+) -> bool:
+    _record_failure(
+        policy_state,
+        resolved_policy,
+        permanent=result.permanent_failure,
+    )
+
+    payload = (
+        _message_payload(current_message)
+        if (current_message := queue.inspect(message.id)) is not None
+        else {"id": message.id, "queue": message.queue}
+    )
+    _print_json(
+        console,
+        {
+            **payload,
+            "state": "failed",
+            "last_error": result.last_error,
+        },
+    )
+    if worker_id is not None:
+        queue.record_worker_heartbeat(worker_id)
+    return forever
+
+
 def _print_dead_letters(
     queue: PersistentQueue,
     *,

localqueue/failure.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any
+from typing import Protocol
 
 _PERMANENT_FAILURE_TYPES = (
     ImportError,
@@ -9,7 +9,11 @@ _PERMANENT_FAILURE_TYPES = (
 )
 
 
-def is_permanent_failure(error: BaseException | Any) -> bool:
+class _ExitCodeCarrier(Protocol):
+    exit_code: int
+
+
+def is_permanent_failure(error: BaseException | _ExitCodeCarrier) -> bool:
     if getattr(error, "exit_code", None) == 127:
         return True
     return isinstance(error, _PERMANENT_FAILURE_TYPES)

localqueue/queue.py

@@ -5,13 +5,15 @@ from pathlib import Path
 from collections.abc import Mapping
 from queue import Empty, Full
 from threading import Condition
-from typing import Any
+from typing import Any, Generic, TypeVar, cast
 
 from .paths import default_queue_store_path
 from .store import QueueMessage, QueueStats, QueueStore, SQLiteQueueStore
 
+T = TypeVar("T")
 
-class PersistentQueue:
+
+class PersistentQueue(Generic[T]):
     name: str
     lease_timeout: float
     maxsize: int
@@ -52,7 +54,7 @@ class PersistentQueue:
 
     def put(
         self,
-        item: Any,
+        item: T,
         block: bool = True,
         timeout: float | None = None,
         *,
@@ -81,7 +83,7 @@ class PersistentQueue:
             _ = self._condition.notify_all()
             return message
 
-    def put_nowait(self, item: Any) -> QueueMessage:
+    def put_nowait(self, item: T) -> QueueMessage:
         return self.put(item, block=False)
 
     def get(
@@ -90,13 +92,13 @@ class PersistentQueue:
         timeout: float | None = None,
         *,
         leased_by: str | None = None,
-    ) -> Any:
+    ) -> T:
         message = self.get_message(block=block, timeout=timeout, leased_by=leased_by)
         with self._condition:
             self._unfinished[message.id] = message
-        return message.value
+        return cast("T", message.value)
 
-    def get_nowait(self) -> Any:
+    def get_nowait(self) -> T:
         return self.get(block=False)
 
     def get_message(

localqueue/retry/store.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Protocol
 
 if TYPE_CHECKING:
+    from types import ModuleType
     import lmdb
 
 _ENVS: dict[tuple[str, int], Any] = {}
@@ -17,7 +18,7 @@ _ENVS_LOCK = threading.Lock()
 _SQLITE_RETRY_SCHEMA_VERSION = 1
 
 
-def _import_lmdb() -> Any:
+def _import_lmdb() -> ModuleType:
     try:
         import lmdb
     except ModuleNotFoundError as exc:

localqueue/retry/tenacity.py

@@ -9,6 +9,7 @@ import weakref
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Callable, TypeVar, cast
+import asyncio
 
 from tenacity import DoAttempt, DoSleep
 from tenacity import AsyncRetrying, RetryCallState, Retrying
@@ -100,7 +101,7 @@ def _get_default_store() -> AttemptStore:
                 _default_owned_stores.add(store)
             except TypeError:
                 pass
-    return cast(AttemptStore, store)
+    return cast("AttemptStore", store)
 
 
 def configure_default_store(store: AttemptStore | None = None) -> None:
@@ -321,7 +322,7 @@ class _PersistentMixin:
         context = getattr(self._local, "persistent_context", None)
         if context is None:
             raise RuntimeError("persistent retry context not initialized")
-        return cast(PersistentCallContext, context)
+        return cast("PersistentCallContext", context)
 
     def _translate_state(self, retry_state: RetryCallState) -> PersistentRetryState:
         context = self._current_context()
@@ -411,7 +412,7 @@ class _PersistentMixin:
 
     @property
     def statistics(self) -> dict[str, Any]:
-        return cast(dict[str, Any], self._retrying.statistics)
+        return cast("dict[str, Any]", self._retrying.statistics)
 
     def begin(self) -> None:
         self._retrying.begin()
@@ -426,7 +427,7 @@ class _PersistentMixin:
         def wrapped_f(*args: Any, **kwargs: Any) -> Any:
             copy = self.copy()
             wrapped_f.statistics = copy.statistics  # type: ignore[attr-defined]
-            return cast(Any, copy)(f, *args, **kwargs)
+            return cast("Any", copy)(f, *args, **kwargs)
 
         def retry_with(*args: Any, **kwargs: Any) -> WrappedFn:
             return self.copy(*args, **kwargs).wraps(f)
@@ -434,7 +435,7 @@ class _PersistentMixin:
         wrapped_f.retry = self  # type: ignore[attr-defined]
         wrapped_f.retry_with = retry_with  # type: ignore[attr-defined]
         wrapped_f.statistics = {}  # type: ignore[attr-defined]
-        return cast(WrappedFn, wrapped_f)
+        return cast("WrappedFn", wrapped_f)
 
     def copy(self, **kwargs: Any) -> "_PersistentMixin":
         store = kwargs.pop("store", self._store)
@@ -449,17 +450,16 @@ class _PersistentMixin:
             _ = tenacity_kwargs.pop("stop", None)
 
         return self.__class__(
-            store=cast(AttemptStore | None, store),
+            store=cast("AttemptStore | None", store),
             store_path=self._store_path
             if store_path is _UNSET
-            else cast(str | Path | None, store_path),
-            key=cast(str | None, key),
+            else cast("str | Path | None", store_path),
+            key=cast("str | None", key),
             key_fn=cast(
-                Callable[[Callable[..., Any], tuple[Any, ...], dict[str, Any]], str]
-                | None,
+                "Callable[[Callable[..., Any], tuple[Any, ...], dict[str, Any]], str] | None",
                 key_fn,
             ),
-            clear_on_success=cast(bool, clear_on_success),
+            clear_on_success=cast("bool", clear_on_success),
             **({} if max_tries is _UNSET else {"max_tries": max_tries}),
             **tenacity_kwargs,
         )
@@ -506,7 +506,10 @@ class PersistentRetrying(_PersistentMixin):
         self._local.persistent_context = context
         try:
             retry_state = RetryCallState(
-                retry_object=cast(Any, self._retrying), fn=fn, args=args, kwargs=kwargs
+                retry_object=cast("Any", self._retrying),
+                fn=fn,
+                args=args,
+                kwargs=kwargs,
             )
             while True:
                 do = self.iter(retry_state=retry_state)
@@ -527,7 +530,7 @@ class PersistentRetrying(_PersistentMixin):
                         retry_state.set_result(result)
                 elif isinstance(do, DoSleep):
                     retry_state.prepare_for_next_attempt()
-                    cast(Any, self._retrying).sleep(do)
+                    cast("Any", self._retrying).sleep(do)
                 else:
                     self._clear_if_success(retry_state)
                     return do
@@ -605,10 +608,13 @@ class PersistentAsyncRetrying(_PersistentMixin):
         if _utils.is_coroutine_callable(self._user_stop):
 
             async def async_wrapped_stop(retry_state: RetryCallState) -> bool:
+                context = self._current_context()
                 should_stop = bool(
                     await self._user_stop(self._translate_state(retry_state))
                 )
-                self._persist_attempt(retry_state, exhausted=should_stop)
+                await self._persist_attempt_async(
+                    context, retry_state, exhausted=should_stop
+                )
                 return should_stop
 
             return async_wrapped_stop
@@ -645,7 +651,8 @@ class PersistentAsyncRetrying(_PersistentMixin):
             async def async_wrapped_retry_error_callback(
                 retry_state: RetryCallState,
             ) -> Any:
-                self._persist_attempt(retry_state, exhausted=True)
+                context = self._current_context()
+                await self._persist_attempt_async(context, retry_state, exhausted=True)
                 return await self._user_retry_error_callback(
                     self._translate_state(retry_state)
                 )
@@ -658,19 +665,66 @@ class PersistentAsyncRetrying(_PersistentMixin):
 
         return wrapped_retry_error_callback
 
+    async def _load_context_async(
+        self, fn: Callable[..., Any], args: tuple[Any, ...], kwargs: dict[str, Any]
+    ) -> PersistentCallContext:
+        return await asyncio.to_thread(self._load_context, fn, args, kwargs)
+
+    async def _persist_attempt_async(
+        self,
+        context: PersistentCallContext,
+        retry_state: RetryCallState,
+        *,
+        exhausted: bool,
+    ) -> None:
+        await asyncio.to_thread(
+            self._persist_attempt_for_context,
+            context,
+            retry_state,
+            exhausted=exhausted,
+        )
+
+    def _persist_attempt_for_context(
+        self,
+        context: PersistentCallContext,
+        retry_state: RetryCallState,
+        *,
+        exhausted: bool,
+    ) -> None:
+        context.record.attempts = context.starting_attempts + retry_state.attempt_number
+        context.record.exhausted = exhausted
+        self._get_store().save(context.key, context.record)
+
+    async def _clear_if_success_async(
+        self, context: PersistentCallContext, retry_state: RetryCallState
+    ) -> None:
+        await asyncio.to_thread(
+            self._clear_if_success_for_context, context, retry_state
+        )
+
+    def _clear_if_success_for_context(
+        self, context: PersistentCallContext, retry_state: RetryCallState
+    ) -> None:
+        outcome = retry_state.outcome
+        if outcome is not None and not outcome.failed:
+            self._get_store().delete(context.key)
+
     async def __call__(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
         self.begin()
-        context = self._load_context(fn, args, kwargs)
+        context = await self._load_context_async(fn, args, kwargs)
         if context.record.exhausted:
             raise PersistentRetryExhausted(context.key, context.record.attempts)
 
         self._local.persistent_context = context
         try:
             retry_state = RetryCallState(
-                retry_object=cast(Any, self._retrying), fn=fn, args=args, kwargs=kwargs
+                retry_object=cast("Any", self._retrying),
+                fn=fn,
+                args=args,
+                kwargs=kwargs,
             )
             while True:
-                do = await cast(AsyncRetrying, self._retrying).iter(
+                do = await cast("AsyncRetrying", self._retrying).iter(
                     retry_state=retry_state
                 )
                 if isinstance(do, DoAttempt):
@@ -692,9 +746,9 @@ class PersistentAsyncRetrying(_PersistentMixin):
                         retry_state.set_result(result)
                 elif isinstance(do, DoSleep):
                     retry_state.prepare_for_next_attempt()
-                    await cast(Any, self._retrying).sleep(do)
+                    await cast("Any", self._retrying).sleep(do)
                 else:
-                    self._clear_if_success(retry_state)
+                    await self._clear_if_success_async(context, retry_state)
                     return do
         finally:
             if hasattr(self._local, "persistent_context"):
@@ -702,10 +756,10 @@ class PersistentAsyncRetrying(_PersistentMixin):
 
 
 def persistent_retry(**kwargs: Any) -> Callable[[WrappedFn], WrappedFn]:
-    return cast(Callable[[WrappedFn], WrappedFn], PersistentRetrying(**kwargs).wraps)
+    return cast("Callable[[WrappedFn], WrappedFn]", PersistentRetrying(**kwargs).wraps)
 
 
 def persistent_async_retry(**kwargs: Any) -> Callable[[WrappedFn], WrappedFn]:
     return cast(
-        Callable[[WrappedFn], WrappedFn], PersistentAsyncRetrying(**kwargs).wraps
+        "Callable[[WrappedFn], WrappedFn]", PersistentAsyncRetrying(**kwargs).wraps
     )

localqueue/store.py

@@ -9,10 +9,11 @@ from collections import Counter
 from contextlib import contextmanager, suppress
 from dataclasses import dataclass, field, replace
 from pathlib import Path
-from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, Iterator, Protocol
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from types import ModuleType
     import lmdb
 
 _ENVS: dict[tuple[str, int], Any] = {}
@@ -24,7 +25,7 @@ _QUEUE_RECORD_VERSION = 4
 _SQLITE_SCHEMA_VERSION = 2
 
 
-def _import_lmdb() -> Any:
+def _import_lmdb() -> ModuleType:
     try:
         import lmdb
     except ModuleNotFoundError as exc:

localqueue/worker.py

@@ -5,13 +5,15 @@ import time
 from dataclasses import dataclass
 from collections.abc import Callable
 from queue import Empty
-from typing import Any, TypeVar, cast
+from typing import Any, TypeVar, cast, TYPE_CHECKING
 
 from .retry import PersistentAsyncRetrying, PersistentRetrying
 
 from .failure import is_permanent_failure
-from .queue import PersistentQueue
-from .store import QueueMessage
+
+if TYPE_CHECKING:
+    from .queue import PersistentQueue
+    from .store import QueueMessage
 
 WrappedFn = TypeVar("WrappedFn", bound=Callable[..., Any])
 _UNSET = object()
@@ -32,9 +34,9 @@ def _resolve_dead_letter_on_failure(
             "dead_letter_on_exhaustion=, not conflicting values"
         )
     if dead_letter_on_failure is not _UNSET:
-        return cast(bool, dead_letter_on_failure)
+        return cast("bool", dead_letter_on_failure)
     if dead_letter_on_exhaustion is not _UNSET:
-        return cast(bool, dead_letter_on_exhaustion)
+        return cast("bool", dead_letter_on_exhaustion)
     return True
 
 
@@ -130,9 +132,9 @@ class PersistentWorkerConfig:
             dead_letter_on_exhaustion=dead_letter_on_exhaustion,
         )
         if release_delay is not _UNSET:
-            _validate_release_delay(cast(float, release_delay))
+            _validate_release_delay(cast("float", release_delay))
         if min_interval is not _UNSET:
-            _validate_min_interval(cast(float, min_interval))
+            _validate_min_interval(cast("float", min_interval))
         if (
             circuit_breaker_failures is not _UNSET
             or circuit_breaker_cooldown is not _UNSET
@@ -140,12 +142,12 @@ class PersistentWorkerConfig:
             resolved_breaker_failures = (
                 self.circuit_breaker_failures
                 if circuit_breaker_failures is _UNSET
-                else cast(int, circuit_breaker_failures)
+                else cast("int", circuit_breaker_failures)
             )
             resolved_breaker_cooldown = (
                 self.circuit_breaker_cooldown
                 if circuit_breaker_cooldown is _UNSET
-                else cast(float, circuit_breaker_cooldown)
+                else cast("float", circuit_breaker_cooldown)
             )
             _validate_circuit_breaker(
                 resolved_breaker_failures, resolved_breaker_cooldown
@@ -160,22 +162,22 @@ class PersistentWorkerConfig:
             release_delay=(
                 self.release_delay
                 if release_delay is _UNSET
-                else cast(float, release_delay)
+                else cast("float", release_delay)
             ),
             min_interval=(
                 self.min_interval
                 if min_interval is _UNSET
-                else cast(float, min_interval)
+                else cast("float", min_interval)
             ),
             circuit_breaker_failures=(
                 self.circuit_breaker_failures
                 if circuit_breaker_failures is _UNSET
-                else cast(int, circuit_breaker_failures)
+                else cast("int", circuit_breaker_failures)
             ),
             circuit_breaker_cooldown=(
                 self.circuit_breaker_cooldown
                 if circuit_breaker_cooldown is _UNSET
-                else cast(float, circuit_breaker_cooldown)
+                else cast("float", circuit_breaker_cooldown)
             ),
             **merged_retry_kwargs,
         )
@@ -313,7 +315,10 @@ def persistent_worker(
                 if worker_id is not None:
                     queue.record_worker_heartbeat(worker_id)
             _record_success(policy_state)
-            queue.ack(message)
+            try:
+                queue.ack(message)
+            except Exception:
+                raise
             return result
 
         return wrapped
@@ -372,7 +377,10 @@ def persistent_async_worker(
                 if worker_id is not None:
                     queue.record_worker_heartbeat(worker_id)
             _record_success(policy_state)
-            queue.ack(message)
+            try:
+                queue.ack(message)
+            except Exception:
+                raise
             return result
 
         return wrapped
@@ -383,6 +391,6 @@ def persistent_async_worker(
 async def _get_message_async(queue: PersistentQueue) -> QueueMessage:
     while True:
         try:
-            return queue.get_message(block=False)
+            return await asyncio.to_thread(queue.get_message, False)
         except Empty:
             await asyncio.sleep(0.05)

{localqueue-0.3.1.dist-info → localqueue-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localqueue
-Version: 0.3.1
+Version: 0.3.2
 Summary: Durable local queues for Python, with persistent retry state powered by Tenacity.
 Project-URL: Homepage, https://brunoportis.github.io/localqueue/
 Project-URL: Documentation, https://brunoportis.github.io/localqueue/
@@ -37,7 +37,8 @@ Description-Content-Type: text/markdown
 
 [![Tests](https://github.com/brunoportis/localqueue/actions/workflows/tests.yml/badge.svg)](https://github.com/brunoportis/localqueue/actions/workflows/tests.yml)
 [![CodeQL](https://github.com/brunoportis/localqueue/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/brunoportis/localqueue/actions/workflows/github-code-scanning/codeql)
-![Coverage](https://img.shields.io/badge/coverage-%E2%89%A595%25-brightgreen)
+[![PyPI version](https://img.shields.io/pypi/v/localqueue.svg)](https://pypi.org/project/localqueue/)
+![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)
 
 `localqueue` is a small durable queue for one machine. It stores work on the local filesystem by default and keeps retry state with Tenacity.
 
@@ -47,6 +48,12 @@ Use it for scripts, CLI tools, cron jobs, and small Python workers that share on
 localqueue queue exec emails -- python scripts/send_email.py
 ```
 
+To run the CLI in a clean container:
+
+```bash
+docker run --rm ghcr.io/brunoportis/localqueue:latest --help
+```
+
 ```python
 from localqueue import PersistentQueue, persistent_worker
 

localqueue-0.3.2.dist-info/RECORD

@@ -0,0 +1,15 @@
+localqueue/__init__.py,sha256=76PTXzQOzD_VWK8sXiPxoyidN3TlxYtg3oJhIDqGBSU,1667
+localqueue/cli.py,sha256=jKNjl45VwjZEzH0r0AI9OvV5SOBLjNhN4x8choZBMSQ,52762
+localqueue/failure.py,sha256=8pBorAaEGoTlij-HDFOTJmNRROsPKY7evMr2rJVeI6w,409
+localqueue/paths.py,sha256=0IRbSMhu35nB5N-HGYltJ0NFYj3I4iagSzUvR15LgWs,555
+localqueue/queue.py,sha256=8M2LnQixnwtD07Oetgugprrn_7q9nVXqhCOAWu6K6JM,10660
+localqueue/store.py,sha256=vxujfXad86gU6SaRSBEvjWXiFJ0KVHESRqUoRgV85ik,72759
+localqueue/worker.py,sha256=AxpVHJciX2bh569kdwqA7oSb_xNQLwg_P-wkQKt4IK8,14010
+localqueue/retry/__init__.py,sha256=kxDYtiksSd-JPlTzmd1esJLBUSrvm2fJD-TN9sMsh0Q,956
+localqueue/retry/store.py,sha256=B9OsmcST_HhGyGYLKgt94_sofBMhAx976sPoX_aykmM,11128
+localqueue/retry/tenacity.py,sha256=s0BmSWUuTZe8Sy5hw_3e46Us94NO8lvuOnrFX-_KJa8,27763
+localqueue-0.3.2.dist-info/METADATA,sha256=i4h_5xrdS9VLLrJrBmyNdPZc_vI8h4kSSrZPOzHp_6Q,3313
+localqueue-0.3.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+localqueue-0.3.2.dist-info/entry_points.txt,sha256=UtW4aZEN4Xi_yb3sFeiAd6k8jQ--1A5acmPs6FAs2JQ,51
+localqueue-0.3.2.dist-info/licenses/LICENSE,sha256=WaGG8H1cJIusbf6T862P7oDXMchv9Gy32I--7WwIdsI,1069
+localqueue-0.3.2.dist-info/RECORD,,

localqueue-0.3.1.dist-info/RECORD

@@ -1,15 +0,0 @@
-localqueue/__init__.py,sha256=76PTXzQOzD_VWK8sXiPxoyidN3TlxYtg3oJhIDqGBSU,1667
-localqueue/cli.py,sha256=we4znzvSI55pobSqhXs1TMm5Yd8k-Y-SRQ71sR03AsA,47464
-localqueue/failure.py,sha256=s9s9q3PqAtsuNn2Huxsd0cYqdery0kHiF0hg92BqEvc,336
-localqueue/paths.py,sha256=0IRbSMhu35nB5N-HGYltJ0NFYj3I4iagSzUvR15LgWs,555
-localqueue/queue.py,sha256=UiATgC8MXtcnQoKNVE9n90eZvWFw-B7vVQ5fBeQQG2o,10603
-localqueue/store.py,sha256=zusZwArzF4aH8N6MuiM-2mwLpvHgaVy8JIQVWZqYfyM,72715
-localqueue/worker.py,sha256=BRdb2d8BrRfKFuhI1ggBHeKOQFkuMvZYy5HuWNOYIXQ,13783
-localqueue/retry/__init__.py,sha256=kxDYtiksSd-JPlTzmd1esJLBUSrvm2fJD-TN9sMsh0Q,956
-localqueue/retry/store.py,sha256=USV1G8RGbThWLwuFpwnr9qntJyctZdEfNKMv21vjTTg,11088
-localqueue/retry/tenacity.py,sha256=cBAoXUesg8aFgRqQtDyL0_j-buKlzdavXSNQ-yFwAdg,25952
-localqueue-0.3.1.dist-info/METADATA,sha256=VRUmcdLZ1ZUnQfWKPSTtX-Eh_cXk9kjm0htD6aWI68s,3107
-localqueue-0.3.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-localqueue-0.3.1.dist-info/entry_points.txt,sha256=UtW4aZEN4Xi_yb3sFeiAd6k8jQ--1A5acmPs6FAs2JQ,51
-localqueue-0.3.1.dist-info/licenses/LICENSE,sha256=WaGG8H1cJIusbf6T862P7oDXMchv9Gy32I--7WwIdsI,1069
-localqueue-0.3.1.dist-info/RECORD,,