localqueue 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

localqueue/__init__.py

@@ -16,6 +16,7 @@ from .retry import (
     PersistentRetrying,
     RetryRecord,
     SQLiteAttemptStore,
+    close_default_store,
     configure_default_store,
     configure_default_store_factory,
     idempotency_key_from_id,
@@ -55,6 +56,7 @@ __all__ = [
     "SQLiteAttemptStore",
     "SQLiteQueueStore",
     "__version__",
+    "close_default_store",
     "configure_default_store",
     "configure_default_store_factory",
     "idempotency_key_from_id",

localqueue/cli.py

@@ -8,17 +8,16 @@ import subprocess
 import sys
 import time
 from collections import Counter
-from collections.abc import Callable
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
 from queue import Empty
-from types import FrameType
-from typing import Any, Iterator
+from typing import Any, Iterator, TYPE_CHECKING
 
 from tenacity import wait_none
 
 from .failure import is_permanent_failure
+from .paths import default_queue_store_path, default_retry_store_path
 from .queue import PersistentQueue
 from .retry import (
     PersistentRetryExhausted,
@@ -34,8 +33,12 @@ from .worker import (
     _sleep_for_policy,
 )
 
-DEFAULT_STORE_PATH = "localqueue_queue.sqlite3"
-DEFAULT_RETRY_STORE_PATH = "localqueue_retries.sqlite3"
+if TYPE_CHECKING:
+    from types import FrameType
+    from collections.abc import Callable
+
+DEFAULT_STORE_PATH = str(default_queue_store_path())
+DEFAULT_RETRY_STORE_PATH = str(default_retry_store_path())
 CONFIG_FILENAME = "config.yaml"
 
 
@@ -113,7 +116,20 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     app.add_typer(queue_app, name="queue")
     app.add_typer(retry_app, name="retry")
     app.add_typer(config_app, name="config")
+    _register_config_commands(config_app, typer, yaml, console, err_console, config)
+    _register_queue_commands(queue_app, typer, console, err_console, config)
+    _register_retry_commands(retry_app, typer, console, err_console, config)
+    return app
 
+
+def _register_config_commands(
+    config_app: Any,
+    typer: Any,
+    yaml: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @config_app.command("path")
     def config_path() -> None:
         console.print(str(_config_path()))
@@ -124,7 +140,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
 
     @config_app.command("init")
     def config_init(
-        store_path: str = typer.Option(DEFAULT_STORE_PATH, "--store-path"),
+        store_path: str | None = typer.Option(None, "--store-path"),
         retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
         dead_letter_ttl_seconds: float | None = typer.Option(
             None, "--dead-letter-ttl-seconds", min=0.0
@@ -141,7 +157,7 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             )
             raise typer.Exit(1)
 
-        new_config: dict[str, Any] = {"store_path": store_path}
+        new_config: dict[str, Any] = {"store_path": _resolve_store_path(store_path, {})}
         if retry_store_path is not None:
             new_config["retry_store_path"] = retry_store_path
         if dead_letter_ttl_seconds is not None:
@@ -180,6 +196,26 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         config.update(updated)
         _print_json(console, config)
 
+
+def _register_queue_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
+    _register_queue_basic_commands(queue_app, typer, console, err_console, config)
+    _register_queue_admin_commands(queue_app, typer, console, err_console, config)
+    _register_queue_worker_commands(queue_app, typer, console, err_console, config)
+
+
+def _register_queue_basic_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("add")
     def queue_add(
         queue: str,
@@ -209,6 +245,17 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             )
         _print_json(console, _message_payload(message))
 
+    _register_queue_message_commands(queue_app, typer, console, err_console, config)
+    _register_queue_maintenance_commands(queue_app, typer, console, err_console, config)
+
+
+def _register_queue_message_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("pop")
     def queue_pop(
         queue: str,
@@ -318,6 +365,27 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
     ) -> None:
         console.print(_queue(queue, _resolve_store_path(store_path, config)).qsize())
 
+
+def _register_queue_maintenance_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
+    _register_queue_stats_commands(queue_app, typer, console, err_console, config)
+    _register_queue_dead_commands(queue_app, typer, console, err_console, config)
+    _register_queue_requeue_commands(queue_app, typer, console, err_console, config)
+    _register_queue_purge_commands(queue_app, typer, console, err_console, config)
+
+
+def _register_queue_stats_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("stats")
     def queue_stats(
         queue: str,
@@ -361,6 +429,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             },
         )
 
+
+def _register_queue_dead_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("dead")
     def queue_dead(
         queue: str,
@@ -433,39 +509,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
                 failed_within=failed_within,
             )
 
-    @retry_app.command("prune")
-    def retry_prune(
-        older_than: float | None = typer.Option(None, "--older-than", min=0.0),
-        retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
-        dry_run: bool = typer.Option(False, "--dry-run"),
-    ) -> None:
-        store = SQLiteAttemptStore(_resolve_retry_store_path(retry_store_path, config))
-        try:
-            resolved_older_than = _resolve_retry_record_ttl(older_than, config)
-            if resolved_older_than is None:
-                err_console.print(
-                    "[red]pass --older-than or configure retry_record_ttl_seconds[/red]"
-                )
-                raise typer.Exit(1)
-            now = time.time()
-            deleted = (
-                store.count_exhausted_older_than(
-                    older_than=resolved_older_than, now=now
-                )
-                if dry_run
-                else store.prune_exhausted(older_than=resolved_older_than, now=now)
-            )
-        finally:
-            store.close()
-        _print_json(
-            console,
-            {
-                "dry_run": dry_run,
-                "older_than": resolved_older_than,
-                "would_delete" if dry_run else "deleted": deleted,
-            },
-        )
 
+def _register_queue_requeue_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("requeue-dead")
     def queue_requeue_dead(
         queue: str,
@@ -517,6 +568,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             )
         _print_json(console, {"id": message_id, "state": "requeued"})
 
+
+def _register_queue_purge_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("purge")
     def queue_purge(
         queue: str,
@@ -528,6 +587,55 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
             _emit_event(err_console, "queue.purge", queue=queue, deleted=deleted)
         console.print(deleted)
 
+
+def _register_retry_commands(
+    retry_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
+    @retry_app.command("prune")
+    def retry_prune(
+        older_than: float | None = typer.Option(None, "--older-than", min=0.0),
+        retry_store_path: str | None = typer.Option(None, "--retry-store-path"),
+        dry_run: bool = typer.Option(False, "--dry-run"),
+    ) -> None:
+        store = SQLiteAttemptStore(_resolve_retry_store_path(retry_store_path, config))
+        try:
+            resolved_older_than = _resolve_retry_record_ttl(older_than, config)
+            if resolved_older_than is None:
+                err_console.print(
+                    "[red]pass --older-than or configure retry_record_ttl_seconds[/red]"
+                )
+                raise typer.Exit(1)
+            now = time.time()
+            deleted = (
+                store.count_exhausted_older_than(
+                    older_than=resolved_older_than, now=now
+                )
+                if dry_run
+                else store.prune_exhausted(older_than=resolved_older_than, now=now)
+            )
+        finally:
+            store.close()
+        _print_json(
+            console,
+            {
+                "dry_run": dry_run,
+                "older_than": resolved_older_than,
+                "would_delete" if dry_run else "deleted": deleted,
+            },
+        )
+
+
+def _register_queue_admin_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("process")
     def queue_process(
         queue: str,
@@ -605,6 +713,14 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         if exit_code:
             raise typer.Exit(exit_code)
 
+
+def _register_queue_worker_commands(
+    queue_app: Any,
+    typer: Any,
+    console: Any,
+    err_console: Any,
+    config: dict[str, Any],
+) -> None:
     @queue_app.command("exec")
     def queue_exec(
         queue: str,
@@ -680,8 +796,6 @@ def _build_app(typer: Any, yaml: Any, console: Any, err_console: Any) -> Any:
         if exit_code:
             raise typer.Exit(exit_code)
 
-    return app
-
 
 def _queue(
     name: str, store_path: str, *, lease_timeout: float = 30.0
@@ -859,91 +973,200 @@ def _process_queue_messages(
     resolved_policy = (
         worker_policy if worker_policy is not None else PersistentWorkerConfig()
     )
+    owned_retry_store: SQLiteAttemptStore | None = None
 
-    while forever or processed < max_jobs:
-        if shutdown.requested:
-            if forever:
-                _print_json(console, {"state": "stopped", "processed": processed})
-            return 0
-
-        if worker_id is not None:
-            queue.record_worker_heartbeat(worker_id)
-        _sleep_for_policy(policy_state, resolved_policy)
+    try:
+        while forever or processed < max_jobs:
+            if shutdown.requested:
+                if forever:
+                    _print_json(console, {"state": "stopped", "processed": processed})
+                return 0
 
-        try:
-            message = queue.get_message(
+            iteration, owned_retry_store, empty_queue = _process_queue_iteration(
+                queue,
+                handler,
+                console=console,
+                err_console=err_console,
+                policy_state=policy_state,
+                resolved_policy=resolved_policy,
+                retry_store_path=retry_store_path,
+                owned_retry_store=owned_retry_store,
+                max_tries=max_tries,
+                worker_id=worker_id,
                 block=block,
-                timeout=_poll_timeout(forever, block, timeout),
-                leased_by=worker_id,
+                timeout=timeout,
+                idle_sleep=idle_sleep,
+                release_delay=release_delay,
+                dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+                log_events=log_events,
+                mode=mode,
+                forever=forever,
             )
-        except Empty:
-            if forever:
-                if not block:
-                    time.sleep(idle_sleep)
-                continue
-            if processed == 0:
-                if err_console is not None:
+            if empty_queue:
+                if processed == 0 and err_console is not None:
                     err_console.print("[yellow]queue is empty[/yellow]")
-                return 1
-            break
+                if processed == 0:
+                    return 1
+                break
+            assert iteration is not None
+            if _handle_processed_queue_result(
+                queue,
+                iteration.message,
+                iteration.process_result,
+                console=console,
+                worker_id=worker_id,
+                policy_state=policy_state,
+            ):
+                processed += 1
+                continue
+            if _handle_failed_queue_result(
+                queue,
+                iteration.message,
+                iteration.process_result,
+                console=console,
+                worker_id=worker_id,
+                policy_state=policy_state,
+                resolved_policy=resolved_policy,
+                forever=forever,
+            ):
+                continue
+            return 1
 
-        if log_events and err_console is not None:
-            _emit_event(
-                err_console,
-                f"{mode}.lease",
-                queue=message.queue,
-                message_id=message.id,
-                leased_by=message.leased_by,
-                attempts=message.attempts,
-                leased_until=message.leased_until,
-            )
+        return 0
+    finally:
+        if owned_retry_store is not None:
+            owned_retry_store.close()
 
-        result = _process_message(
-            queue,
-            message,
-            handler,
-            retry_store_path=retry_store_path,
-            max_tries=max_tries,
-            release_delay=release_delay,
-            dead_letter_on_exhaustion=dead_letter_on_exhaustion,
-            log_events=log_events,
-            mode=mode,
-            err_console=err_console,
-        )
-        if result.processed:
-            processed += 1
-            _record_success(policy_state)
-            if worker_id is not None:
-                queue.record_worker_heartbeat(worker_id)
-            _print_json(console, {"id": message.id, "state": "acked"})
-            continue
 
-        _record_failure(
-            policy_state,
-            resolved_policy,
-            permanent=result.permanent_failure,
-        )
+@dataclass(slots=True)
+class _QueueIterationResult:
+    message: QueueMessage
+    process_result: _ProcessResult
 
-        payload = (
-            _message_payload(current_message)
-            if (current_message := queue.inspect(message.id)) is not None
-            else {"id": message.id, "queue": message.queue}
-        )
-        _print_json(
-            console,
-            {
-                **payload,
-                "state": "failed",
-                "last_error": result.last_error,
-            },
+
+def _process_queue_iteration(
+    queue: PersistentQueue,
+    handler: Callable[[Any], Any],
+    *,
+    console: Any,
+    err_console: Any,
+    policy_state: WorkerPolicyState,
+    resolved_policy: PersistentWorkerConfig,
+    retry_store_path: str | None,
+    owned_retry_store: SQLiteAttemptStore | None,
+    max_tries: int,
+    worker_id: str | None,
+    block: bool,
+    timeout: float | None,
+    idle_sleep: float,
+    release_delay: float,
+    dead_letter_on_exhaustion: bool,
+    log_events: bool,
+    mode: str,
+    forever: bool,
+) -> tuple[_QueueIterationResult | None, SQLiteAttemptStore | None, bool]:
+    if worker_id is not None:
+        queue.record_worker_heartbeat(worker_id)
+    _sleep_for_policy(policy_state, resolved_policy)
+
+    try:
+        message = queue.get_message(
+            block=block,
+            timeout=_poll_timeout(forever, block, timeout),
+            leased_by=worker_id,
         )
-        if worker_id is not None:
-            queue.record_worker_heartbeat(worker_id)
+    except Empty:
         if forever:
-            continue
-        return 1
+            if not block:
+                time.sleep(idle_sleep)
+            return None, owned_retry_store, True
+        return None, owned_retry_store, True
+
+    if log_events and err_console is not None:
+        _emit_event(
+            err_console,
+            f"{mode}.lease",
+            queue=message.queue,
+            message_id=message.id,
+            leased_by=message.leased_by,
+            attempts=message.attempts,
+            leased_until=message.leased_until,
+        )
+
+    if retry_store_path is not None and owned_retry_store is None:
+        owned_retry_store = SQLiteAttemptStore(retry_store_path)
+
+    result = _process_message(
+        queue,
+        message,
+        handler,
+        retry_store=owned_retry_store,
+        retry_store_path=None if owned_retry_store is not None else retry_store_path,
+        max_tries=max_tries,
+        release_delay=release_delay,
+        dead_letter_on_exhaustion=dead_letter_on_exhaustion,
+        log_events=log_events,
+        mode=mode,
+        err_console=err_console,
+    )
+    return (
+        _QueueIterationResult(message=message, process_result=result),
+        owned_retry_store,
+        False,
+    )
 
-    return 0
+
+def _handle_processed_queue_result(
+    queue: PersistentQueue,
+    message: QueueMessage,
+    result: _ProcessResult,
+    *,
+    console: Any,
+    worker_id: str | None,
+    policy_state: WorkerPolicyState,
+) -> bool:
+    if not result.processed:
+        return False
+    _record_success(policy_state)
+    if worker_id is not None:
+        queue.record_worker_heartbeat(worker_id)
+    _print_json(console, {"id": message.id, "state": "acked"})
+    return True
+
+
+def _handle_failed_queue_result(
+    queue: PersistentQueue,
+    message: QueueMessage,
+    result: _ProcessResult,
+    *,
+    console: Any,
+    worker_id: str | None,
+    policy_state: WorkerPolicyState,
+    resolved_policy: PersistentWorkerConfig,
+    forever: bool,
+) -> bool:
+    _record_failure(
+        policy_state,
+        resolved_policy,
+        permanent=result.permanent_failure,
+    )
+
+    payload = (
+        _message_payload(current_message)
+        if (current_message := queue.inspect(message.id)) is not None
+        else {"id": message.id, "queue": message.queue}
+    )
+    _print_json(
+        console,
+        {
+            **payload,
+            "state": "failed",
+            "last_error": result.last_error,
+        },
+    )
+    if worker_id is not None:
+        queue.record_worker_heartbeat(worker_id)
+    return forever
 
 
 def _print_dead_letters(
@@ -1122,13 +1345,13 @@ def _write_config(yaml: Any, config: dict[str, Any], path: Path | None = None) -
 
 
 def _resolve_store_path(explicit: str | None, config: dict[str, Any]) -> str:
-    return str(explicit or config.get("store_path") or DEFAULT_STORE_PATH)
+    return str(explicit or config.get("store_path") or default_queue_store_path())
 
 
 def _resolve_retry_store_path(explicit: str | None, config: dict[str, Any]) -> str:
     value = explicit if explicit is not None else config.get("retry_store_path")
     if value is None:
-        return DEFAULT_RETRY_STORE_PATH
+        return str(default_retry_store_path())
     return str(value)
 
 

localqueue/failure.py

@@ -1,27 +1,19 @@
 from __future__ import annotations
 
-from typing import Any
+from typing import Protocol
 
 _PERMANENT_FAILURE_TYPES = (
-    AssertionError,
-    ArithmeticError,
-    AttributeError,
-    FileNotFoundError,
     ImportError,
-    IndexError,
-    IsADirectoryError,
-    KeyError,
-    LookupError,
     ModuleNotFoundError,
     NameError,
-    NotADirectoryError,
-    TypeError,
-    UnicodeError,
-    ValueError,
 )
 
 
-def is_permanent_failure(error: BaseException | Any) -> bool:
+class _ExitCodeCarrier(Protocol):
+    exit_code: int
+
+
+def is_permanent_failure(error: BaseException | _ExitCodeCarrier) -> bool:
     if getattr(error, "exit_code", None) == 127:
         return True
     return isinstance(error, _PERMANENT_FAILURE_TYPES)

localqueue/paths.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+APP_DIR_NAME = "localqueue"
+DEFAULT_QUEUE_DB_NAME = "queue.sqlite3"
+DEFAULT_RETRY_DB_NAME = "retries.sqlite3"
+
+
+def data_dir() -> Path:
+    data_home = os.environ.get("XDG_DATA_HOME")
+    if data_home:
+        return Path(data_home) / APP_DIR_NAME
+    return Path.home() / ".local" / "share" / APP_DIR_NAME
+
+
+def default_queue_store_path() -> Path:
+    return data_dir() / DEFAULT_QUEUE_DB_NAME
+
+
+def default_retry_store_path() -> Path:
+    return data_dir() / DEFAULT_RETRY_DB_NAME