localqueue 0.3.0__tar.gz → 0.3.1__tar.gz

{localqueue-0.3.0 → localqueue-0.3.1}/.gitignore

@@ -17,4 +17,6 @@ site/
 /localqueue_queue.sqlite3
 /localqueue_retries
 /localqueue_retries.sqlite3
+/examples/*.sqlite3
+/examples/*.sqlite3-*
 feedback.md

{localqueue-0.3.0 → localqueue-0.3.1}/CHANGELOG.md

@@ -1,6 +1,17 @@
 # Changelog
 
-## Unreleased
+## 0.3.1
+
+- Use XDG data directories for default SQLite queue and retry store files.
+- Move the exploratory retry demo from `main.py` to `examples/retry_demo.py`.
+- Make `PersistentQueue` unfinished-message tracking O(1) by message id.
+- Add an end-to-end SQLite worker test covering queue ack and retry cleanup.
+- Make `store_path=` for persistent retries open a SQLite attempt store.
+- Tighten permanent-failure classification to avoid dead-lettering common
+  transient application errors such as `ValueError` and `KeyError`.
+- Optimize SQLite queue stats and dead-letter retention queries with
+  materialized columns and schema migration support.
+- Add indexed SQLite retry-store retention for exhausted records.
 
 ## 0.3.0
 

{localqueue-0.3.0 → localqueue-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localqueue
-Version: 0.3.0
+Version: 0.3.1
 Summary: Durable local queues for Python, with persistent retry state powered by Tenacity.
 Project-URL: Homepage, https://brunoportis.github.io/localqueue/
 Project-URL: Documentation, https://brunoportis.github.io/localqueue/

{localqueue-0.3.0 → localqueue-0.3.1}/docs/api.md

@@ -21,7 +21,7 @@ Constructor options:
 | --- | --- |
 | `name` | queue name inside the store |
 | `store` | queue store instance |
-| `store_path` | path for the default SQLite queue store |
+| `store_path` | explicit path for the SQLite queue store |
 | `lease_timeout` | seconds before an inflight message is redelivered |
 | `maxsize` | maximum number of ready messages; `0` means unbounded |
 | `retry_defaults` | Tenacity retry keyword defaults inherited by workers |
@@ -183,8 +183,8 @@ SQLite-backed queue store. This is the default backend. Records are serialized
 as versioned JSON; values must be JSON-serializable.
 
 The SQLite store tracks its on-disk schema version with `PRAGMA user_version`.
-Current releases accept older compatible versions and reject future versions
-until a migration is defined.
+Current releases migrate older compatible versions and reject future versions
+they do not know how to migrate yet.
 
 #### `LMDBQueueStore`
 
@@ -237,7 +237,7 @@ Both decorators require `key=` or `key_fn=` and accept the persistent options be
 | Option | Meaning |
 | --- | --- |
 | `store` | attempt store instance |
-| `store_path` | path for an LMDB attempt-store directory |
+| `store_path` | explicit path for a SQLite attempt-store file |
 | `key` | fixed retry key |
 | `key_fn` | function that derives a retry key from the call |
 | `clear_on_success` | delete the attempt record after success |
@@ -341,6 +341,11 @@ Dataclass stored per retry key.
 | `first_attempt_at` | `float` | Unix timestamp of the first persisted attempt |
 | `exhausted` | `bool` | whether the retry budget is exhausted |
 
+#### `close_default_store(all_threads=False)`
+
+Close the default retry store for the current thread. Pass `all_threads=True`
+at process shutdown to close known factory-created default stores.
+
 #### `AttemptStore`
 
 Protocol for custom attempt stores.
@@ -365,8 +370,9 @@ store = LMDBAttemptStore("/var/lib/my-worker/retries")
 
 #### `SQLiteAttemptStore`
 
-SQLite-backed attempt store. This is the default backend and does not require LMDB's
-native dependency.
+SQLite-backed attempt store. This is the default backend and does not require
+LMDB's native dependency. Retention cleanup uses indexed `exhausted` and
+`first_attempt_at` columns instead of scanning every serialized retry record.
 
 ```python
 from localqueue.retry import SQLiteAttemptStore

{localqueue-0.3.0 → localqueue-0.3.1}/docs/operational-maturity.md

@@ -107,7 +107,7 @@ inspection tools for that local model, not a distributed liveness protocol.
   versions.
 
 The SQLite store keeps a schema version in `PRAGMA user_version`. Current
-releases accept older compatible versions and reject future versions they do
+releases migrate older compatible versions and reject future versions they do
 not know how to migrate yet. That keeps upgrades explicit and leaves a clear
 path for future schema changes.
 

{localqueue-0.3.0 → localqueue-0.3.1}/docs/queues.md

@@ -23,7 +23,6 @@ from localqueue import PersistentQueue
 
 queue = PersistentQueue(
     "emails",
-    store_path="./localqueue_queue.sqlite3",
     lease_timeout=30.0,
     retry_defaults={
         "max_tries": 3,
@@ -75,10 +74,12 @@ For shared queue-wide retry defaults, attach `retry_defaults` to the
 `PersistentQueue` and let workers inherit those Tenacity arguments unless the
 worker passes explicit overrides.
 
-`localqueue` treats validation-style failures such as `ValueError`, `TypeError`,
-`KeyError`, `IndexError`, and missing command execution as permanent failures.
-Those are dead-lettered even when the worker is configured to release on final
-failure.
+`localqueue` keeps built-in permanent-failure classification conservative.
+Import/name-resolution failures and missing command execution are dead-lettered
+even when the worker is configured to release on final failure. Runtime
+validation errors such as `ValueError`, `TypeError`, and `KeyError` follow the
+configured worker policy because they can represent either bad input or a
+recoverable transient condition.
 
 Change that behavior with `dead_letter_on_failure=False`. The older
 `dead_letter_on_exhaustion` name is still accepted as a compatibility alias.
@@ -332,7 +333,12 @@ Negative delays raise `ValueError`.
 
 ## Stores
 
-The default queue store is SQLite at `./localqueue_queue.sqlite3`.
+The default queue store is SQLite at
+`$XDG_DATA_HOME/localqueue/queue.sqlite3`. When `XDG_DATA_HOME` is not set,
+the fallback is `~/.local/share/localqueue/queue.sqlite3`.
+
+Pass `store_path=` when the queue file must live somewhere explicit, such as a
+service data directory.
 
 ```python
 from localqueue import PersistentQueue, SQLiteQueueStore
@@ -404,7 +410,7 @@ SQLite files can keep running for a long time, but they still need normal file
 maintenance.
 
 The SQLite store keeps a schema version in `PRAGMA user_version`. Current
-releases accept older compatible versions and reject future versions they do
+releases migrate older compatible versions and reject future versions they do
 not know how to migrate yet.
 
 - Keep the main `*.sqlite3` file and its retry store together when you back up

{localqueue-0.3.0 → localqueue-0.3.1}/docs/retries.md

@@ -222,29 +222,29 @@ except PersistentRetryExhausted as exc:
 
 ## Stores
 
-The default attempt store is SQLite at `./localqueue_retries.sqlite3`.
+The default attempt store is SQLite at
+`$XDG_DATA_HOME/localqueue/retries.sqlite3`. When `XDG_DATA_HOME` is not set,
+the fallback is `~/.local/share/localqueue/retries.sqlite3`.
 
-`store_path=` creates an LMDB attempt-store directory. Install
-`localqueue[lmdb]` and use it when you want the retry API to manage an LMDB
-backend directly.
+`store_path=` creates a SQLite attempt-store file, matching the default backend.
 
 ```python
 from localqueue.retry import PersistentRetrying, key_from_argument
 
 retryer = PersistentRetrying(
-    store_path="/var/lib/my-worker/retries",
+    store_path="/var/lib/my-worker/retries.sqlite3",
     max_tries=5,
     key_fn=key_from_argument("job_id"),
 )
 ```
 
-Provide a store instance with `store=` when you need SQLite, full control, or
-in-memory tests.
+Provide a store instance with `store=` when you need LMDB, full control, or
+in-memory tests. Install `localqueue[lmdb]` when you want the LMDB backend.
 
 ```python
-from localqueue.retry import SQLiteAttemptStore, key_from_argument, persistent_retry
+from localqueue.retry import LMDBAttemptStore, key_from_argument, persistent_retry
 
-store = SQLiteAttemptStore("/var/lib/my-worker/retries.sqlite3")
+store = LMDBAttemptStore("/var/lib/my-worker/retries")
 
 
 @persistent_retry(
@@ -257,7 +257,8 @@ def flaky(job_id: str) -> str:
 ```
 
 The CLI `retry_store_path` setting and `queue process --retry-store-path` use a
-SQLite file path.
+SQLite file path. `queue process` and `queue exec` keep one retry-store
+connection open for the worker loop and close it when the loop exits.
 
 ```python
 from localqueue.retry import MemoryAttemptStore, key_from_argument, persistent_retry
@@ -277,6 +278,11 @@ def flaky(job_id: str) -> str:
 
 `store=` and `store_path=` are mutually exclusive.
 
+Default retry stores are thread-local. Call `close_default_store()` when a
+long-lived thread is shutting down and you want to close its default SQLite
+connection explicitly. At process shutdown, `close_default_store(all_threads=True)`
+closes known factory-created default stores.
+
 ## State and callbacks
 
 Tenacity callbacks and strategies receive a state object that behaves like Tenacity's `RetryCallState`, with persistent attempt numbering.

{localqueue-0.3.0 → localqueue-0.3.1}/examples/enqueue_email.py

@@ -8,11 +8,15 @@ from localqueue import PersistentQueue
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("address")
-    parser.add_argument("--store-path", default="./localqueue_queue.sqlite3")
+    parser.add_argument("--store-path")
     parser.add_argument("--fail", action="store_true")
     args = parser.parse_args()
 
-    queue = PersistentQueue("emails", store_path=args.store_path)
+    queue = (
+        PersistentQueue("emails", store_path=args.store_path)
+        if args.store_path is not None
+        else PersistentQueue("emails")
+    )
     message = queue.put({"to": args.address, "fail": args.fail})
     print(message.id)
 

localqueue-0.3.1/examples/retry_demo.py

@@ -0,0 +1,278 @@
+import argparse
+import random
+import time
+from queue import Empty
+from typing import Callable
+
+from localqueue.retry import (
+    AttemptStoreLockedError,
+    PersistentRetrying,
+    PersistentRetryExhausted,
+    key_from_argument,
+    persistent_retry,
+)
+from localqueue import PersistentQueue, QueueStoreLockedError
+from tenacity import wait_exponential, wait_none
+
+
+SendEmailFn = Callable[[str, str], None]
+ProcessVideoFn = Callable[[str, str], None]
+WorkerJobFn = Callable[[str, dict[str, str]], None]
+DEFAULT_EMAIL_FAILURE_RATE = 0.8
+DEFAULT_VIDEO_FAILURE_RATE = 0.7
+DEFAULT_WORKER_FAILURE_RATE = 0.65
+
+
+def build_fast_examples(
+    db_path: str,
+    *,
+    email_failure_rate: float = DEFAULT_EMAIL_FAILURE_RATE,
+    video_failure_rate: float = DEFAULT_VIDEO_FAILURE_RATE,
+) -> tuple[SendEmailFn, ProcessVideoFn]:
+    @persistent_retry(
+        max_tries=3,
+        store_path=db_path,
+        wait=wait_none(),
+        key_fn=key_from_argument("task_id"),
+    )
+    def send_email_fast(task_id: str, email_address: str) -> None:
+        print(f"[send_email_fast] task={task_id} email={email_address}")
+        if random.random() < email_failure_rate:
+            raise ConnectionError("mail server down")
+        print(f"[send_email_fast] task={task_id} sent")
+
+    @persistent_retry(
+        max_tries=5,
+        store_path=db_path,
+        wait=wait_none(),
+        key_fn=key_from_argument("task_id"),
+    )
+    def process_video_fast(task_id: str, file_path: str) -> None:
+        print(f"[process_video_fast] task={task_id} file={file_path}")
+        if random.random() < video_failure_rate:
+            raise ValueError("transcoder busy")
+        print(f"[process_video_fast] task={task_id} processed")
+
+    return send_email_fast, process_video_fast
+
+
+def build_slow_examples(
+    db_path: str,
+    *,
+    email_failure_rate: float = DEFAULT_EMAIL_FAILURE_RATE,
+    video_failure_rate: float = DEFAULT_VIDEO_FAILURE_RATE,
+) -> tuple[SendEmailFn, ProcessVideoFn]:
+    @persistent_retry(
+        max_tries=3,
+        store_path=db_path,
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        key_fn=key_from_argument("task_id"),
+    )
+    def send_email_slow(task_id: str, email_address: str) -> None:
+        print(f"[send_email_slow] task={task_id} email={email_address}")
+        if random.random() < email_failure_rate:
+            raise ConnectionError("mail server down")
+        print(f"[send_email_slow] task={task_id} sent")
+
+    @persistent_retry(
+        max_tries=5,
+        store_path=db_path,
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        key_fn=key_from_argument("task_id"),
+    )
+    def process_video_slow(task_id: str, file_path: str) -> None:
+        print(f"[process_video_slow] task={task_id} file={file_path}")
+        if random.random() < video_failure_rate:
+            raise ValueError("transcoder busy")
+        print(f"[process_video_slow] task={task_id} processed")
+
+    return send_email_slow, process_video_slow
+
+
+def build_worker_example(
+    db_path: str, *, worker_failure_rate: float = DEFAULT_WORKER_FAILURE_RATE
+) -> WorkerJobFn:
+    @persistent_retry(
+        max_tries=4,
+        store_path=db_path,
+        wait=wait_none(),
+        key_fn=key_from_argument("task_id"),
+    )
+    def process_worker_job(task_id: str, payload: dict[str, str]) -> None:
+        print(
+            f"[worker] job={task_id} kind={payload['kind']} target={payload['target']}"
+        )
+        if random.random() < worker_failure_rate:
+            raise RuntimeError("transient worker failure")
+        print(f"[worker] job={task_id} completed")
+
+    return process_worker_job
+
+
+def run_pair(
+    prefix: str, send_email_fn: SendEmailFn, process_video_fn: ProcessVideoFn
+) -> None:
+    print(f"\n=== {prefix} ===")
+    for i in range(1, 4):
+        email_task_id = f"{prefix}:send-email:{i}"
+        video_task_id = f"{prefix}:process-video:{i}"
+
+        try:
+            _ = send_email_fn(email_task_id, "user@example.com")
+        except PersistentRetryExhausted as exc:
+            print(f"[{prefix}] {exc.key} exhausted after {exc.attempts} attempts")
+        except Exception as exc:
+            print(f"[{prefix}] {email_task_id} failed this run: {exc}")
+
+        try:
+            _ = process_video_fn(video_task_id, f"video_{i}.mp4")
+        except PersistentRetryExhausted as exc:
+            print(f"[{prefix}] {exc.key} exhausted after {exc.attempts} attempts")
+        except Exception as exc:
+            print(f"[{prefix}] {video_task_id} failed this run: {exc}")
+
+
+def run_demo(
+    db_path: str,
+    *,
+    email_failure_rate: float = DEFAULT_EMAIL_FAILURE_RATE,
+    video_failure_rate: float = DEFAULT_VIDEO_FAILURE_RATE,
+) -> None:
+    send_email_fast, process_video_fast = build_fast_examples(
+        db_path,
+        email_failure_rate=email_failure_rate,
+        video_failure_rate=video_failure_rate,
+    )
+    send_email_slow, process_video_slow = build_slow_examples(
+        db_path,
+        email_failure_rate=email_failure_rate,
+        video_failure_rate=video_failure_rate,
+    )
+    run_pair("fast", send_email_fast, process_video_fast)
+    run_pair("slow", send_email_slow, process_video_slow)
+
+
+def run_worker_demo(
+    db_path: str, *, worker_failure_rate: float = DEFAULT_WORKER_FAILURE_RATE
+) -> None:
+    print("\n=== persistent queue worker ===")
+    queue = PersistentQueue(
+        "worker-jobs",
+        store_path=f"{db_path}_queue",
+        lease_timeout=2,
+    )
+    retryer = PersistentRetrying(
+        store_path=f"{db_path}_queue_retries",
+        max_tries=4,
+        wait=wait_none(),
+        key_fn=key_from_argument("task_id"),
+    )
+
+    if queue.empty():
+        print("[queue] enqueue sample jobs")
+        _ = queue.put({"kind": "email", "target": "alpha@example.com"})
+        _ = queue.put({"kind": "video", "target": "video_2.mp4"})
+        _ = queue.put({"kind": "webhook", "target": "customer-3"}, delay=1)
+        _ = queue.put({"kind": "invoice", "target": "invoice-4"}, delay=2)
+    else:
+        print(f"[queue] resuming {queue.qsize()} ready job(s)")
+
+    def process_worker_job(task_id: str, payload: dict[str, str]) -> None:
+        print(
+            f"[worker] message={task_id} kind={payload['kind']} "
+            + f"target={payload['target']}"
+        )
+        if random.random() < worker_failure_rate:
+            raise RuntimeError("transient worker failure")
+        print(f"[worker] message={task_id} completed")
+
+    for cycle in range(1, 8):
+        print(f"\n[worker] poll cycle {cycle}")
+        processed = 0
+
+        while True:
+            try:
+                message = queue.get_message(block=False)
+            except Empty:
+                break
+
+            payload = message.value
+            try:
+                _ = retryer(process_worker_job, message.id, payload)
+            except PersistentRetryExhausted as exc:
+                _ = queue.dead_letter(message)
+                print(
+                    f"[queue] message={exc.key} moved to dead letter "
+                    + f"after {exc.attempts} attempts"
+                )
+            except Exception as exc:
+                _ = queue.release(message, delay=1)
+                print(f"[queue] message={message.id} released for retry: {exc}")
+            else:
+                _ = queue.ack(message)
+                print(f"[queue] message={message.id} acked")
+            processed += 1
+
+        if processed == 0:
+            print("[worker] no tasks available")
+        time.sleep(1)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="localqueue demo CLI; queue code lives in localqueue/ and retry code in localqueue/retry/"
+    )
+    _ = parser.add_argument(
+        "--worker",
+        action="store_true",
+        help="run a worker-style example instead of the fast/slow demo",
+    )
+    _ = parser.add_argument(
+        "--db-path",
+        default="examples/localqueue_retry_demo.sqlite3",
+        help="SQLite file for this demo's persisted retry state",
+    )
+    _ = parser.add_argument(
+        "--email-failure-rate",
+        type=float,
+        default=DEFAULT_EMAIL_FAILURE_RATE,
+        help="demo-only probability that email jobs fail",
+    )
+    _ = parser.add_argument(
+        "--video-failure-rate",
+        type=float,
+        default=DEFAULT_VIDEO_FAILURE_RATE,
+        help="demo-only probability that video jobs fail",
+    )
+    _ = parser.add_argument(
+        "--worker-failure-rate",
+        type=float,
+        default=DEFAULT_WORKER_FAILURE_RATE,
+        help="demo-only probability that worker jobs fail",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        if args.worker:
+            run_worker_demo(args.db_path, worker_failure_rate=args.worker_failure_rate)
+        else:
+            run_demo(
+                args.db_path,
+                email_failure_rate=args.email_failure_rate,
+                video_failure_rate=args.video_failure_rate,
+            )
+    except (AttemptStoreLockedError, QueueStoreLockedError) as exc:
+        print(exc)
+        print(
+            "Tip: rerun with --db-path pointing at a different file, "
+            + "for example 'examples/localqueue_retry_demo_2.sqlite3'."
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{localqueue-0.3.0 → localqueue-0.3.1}/localqueue/__init__.py

@@ -16,6 +16,7 @@ from .retry import (
     PersistentRetrying,
     RetryRecord,
     SQLiteAttemptStore,
+    close_default_store,
     configure_default_store,
     configure_default_store_factory,
     idempotency_key_from_id,
@@ -55,6 +56,7 @@ __all__ = [
     "SQLiteAttemptStore",
     "SQLiteQueueStore",
     "__version__",
+    "close_default_store",
     "configure_default_store",
     "configure_default_store_factory",
     "idempotency_key_from_id",