iddqueue 0.13.0rc1__tar.gz → 0.13.0rc3__tar.gz

iddqueue-0.13.0rc3/PKG-INFO

@@ -0,0 +1,183 @@
+Metadata-Version: 2.4
+Name: iddqueue
+Version: 0.13.0rc3
+Summary: Postgres Broker for Dramatiq Task Queue
+Keywords: postgres,task queue,dramatiq
+Author: Étienne BERSAC
+License-Expression: PostgreSQL
+License-File: LICENSE
+Requires-Dist: dramatiq>=2.2.1,<3
+Requires-Dist: psycopg>=3.3.6,<4
+Requires-Dist: psycopg-pool>=3.3.3,<4
+Requires-Dist: tenacity>=9,<10
+Requires-Dist: psycopg[binary]>=3.3.6,<4 ; extra == 'binary'
+Requires-Dist: dramatiq[prometheus]>=2.2.1,<3 ; extra == 'monitoring'
+Requires-Python: >=3.10, <4
+Project-URL: Repository, https://github.com/wa-pis/iddqueue
+Project-URL: Issues, https://github.com/wa-pis/iddqueue/issues
+Provides-Extra: binary
+Provides-Extra: monitoring
+Description-Content-Type: text/markdown
+
+# IDDQueue
+
+[Documentation](https://wa-pis.github.io/iddqueue/) · [Step-by-step setup](https://wa-pis.github.io/iddqueue/walkthrough/)
+
+A PostgreSQL broker and Results backend for [Dramatiq](https://dramatiq.io/),
+built on synchronous Psycopg 3. Tasks, results and coordination live in PostgreSQL;
+no Redis or ORM is required by IDDQueue.
+
+**Current release: [0.13.0rc3](https://pypi.org/project/iddqueue/0.13.0rc3/)** —
+a release candidate for evaluation. [GitHub assets](https://github.com/wa-pis/iddqueue/releases/tag/v0.13.0rc3)
+include the wheel, sdist and SHA256 checksums.
+
+IDDQueue is a fork of [DALIBO's dramatiq-pg](https://gitlab.com/dalibo/dramatiq-pg)
+([original package](https://pypi.org/project/dramatiq-pg/)). It preserves the
+PostgreSQL license and original contributor credits.
+
+## What it provides
+
+- Durable JSONB task/results storage, delayed tasks and crash recovery.
+- Transactional publication alongside business writes; atomic batches and deduplication.
+- PostgreSQL rate limiters, barriers and Dramatiq group callbacks.
+- Pause/resume, cooperative cancellation, failed-task inspection/retry and attempt history.
+- Fixed-interval scheduling and optional Prometheus queue metrics.
+- Schema/prefix isolation and a [tested FastAPI example](docs/fastapi.md).
+
+Delivery is **at least once**: actors and callbacks must be idempotent.
+LISTEN/NOTIFY carries message IDs as wakeup hints; consumers claim authoritative
+SQL rows with session advisory locks. See [behavior and limits](docs/user-guide.md).
+
+## Install
+
+Requires Python 3.10+, Dramatiq 2.2.1+ and Psycopg 3.3.6+.
+CI tests Python **3.10 / 3.13 / 3.14** with PostgreSQL **14 / 18**.
+See [compatibility and API stability](SUPPORT.md).
+
+```sh
+uv venv
+uv pip install "iddqueue[binary]==0.13.0rc3"
+```
+
+The `binary` extra supplies libpq. Base installs require system libpq;
+`[binary,monitoring]` also enables Prometheus support.
+
+**Migrating from dramatiq-pg?** Follow the [migration guide](docs/migration.md)
+before changing imports, pools or database storage. Stop all participants,
+back up data, run the upgrade and restart consistently. Fresh initialization
+and an existing-storage upgrade are different operations.
+
+## Quickstart
+
+Configure PostgreSQL using `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE` and your
+normal credential mechanism. Initialize fresh storage:
+
+```sh
+uv run --no-sync iddqueue init
+```
+
+For existing storage, use `iddqueue upgrade` after following the migration guide.
+Both commands manage the complete feature schema.
+
+Save as `tasks.py`:
+
+```python
+import dramatiq
+from iddqueue import PostgresBroker
+
+broker = PostgresBroker()
+dramatiq.set_broker(broker)
+
+@dramatiq.actor(broker=broker, store_results=True)
+def add(left: int, right: int):
+    return left + right
+```
+
+Start a separate worker from the same directory/environment:
+
+```sh
+uv run --no-sync dramatiq --use-spawn --processes=1 --threads=1 tasks
+```
+
+In another terminal, publish and read the result:
+
+```sh
+uv run --no-sync python - <<'PY'
+from tasks import add, broker
+
+try:
+    message = add.send(2, 3)
+    print(message.message_id)
+    print(message.get_result(backend=broker.backend, block=True, timeout=10000))
+finally:
+    broker.close()
+PY
+```
+
+The result is `5`; timeouts are milliseconds. Each process creates its own pool.
+Close broker-owned pools on shutdown; callers close pools they supply.
+
+## Guides and examples
+
+[Domain actors and one worker bootstrap](docs/domains.md) are included since RC3.
+
+- [Documentation index](docs/index.md), [user guide](docs/user-guide.md), [API](docs/api.md).
+- [Detailed recipes](docs/recipes.md): transactions, middleware, controls, metrics and schedules.
+- [FastAPI](docs/fastapi.md): lifespan, async endpoint thread offload and separate worker.
+- [Deployment and retention](docs/deployment-guide.md), [why PostgreSQL](docs/why.md).
+- [RC notes](docs/rc-0.13.0rc3.md), [changelog](CHANGELOG.md), [release checks](docs/release.md).
+
+The broker and result APIs are synchronous. Async actors use Dramatiq's AsyncIO
+middleware; FastAPI async endpoints offload publication to a thread.
+Transactional enqueue does not accept a Psycopg AsyncConnection.
+
+Session-bound locks/listeners require persistent database sessions; PgBouncer
+transaction pooling is unsuitable. Namespace separation is not access control.
+The legacy [django-dramatiq-pg](https://github.com/uptick/django-dramatiq-pg/)
+integration has been archived since September 3, 2024; IDDQueue compatibility
+is unverified. No tested Django integration is currently provided.
+
+## Compared with dramatiq-pg
+
+Baseline: [dramatiq-pg 0.12.0](https://pypi.org/project/dramatiq-pg/0.12.0/),
+compared with the published IDDQueue 0.13.0rc3 prerelease. This describes the published version, not every
+future upstream revision. Both projects provide a PostgreSQL Dramatiq broker.
+
+| Area | dramatiq-pg 0.12.0 | IDDQueue 0.13.0rc3 / practical benefit |
+| --- | --- | --- |
+| Core storage and delivery | JSONB tasks/results, delayed tasks, LISTEN/NOTIFY, advisory locks, recovery and maintenance CLI | Preserved; at-least-once delivery still requires idempotent actors |
+| Actor initialization | Broker configured before actor imports | Opt-in Domain declarations import before DSN; explicit startup registration and domain queues |
+| Runtime | Python >=3.6,<4; Dramatiq >=1.5,<2; Psycopg 2 | Python >=3.10,<4; Dramatiq >=2.2.1,<3; synchronous Psycopg 3 and psycopg-pool |
+| Pool lifecycle | Psycopg 2 pools | Lazy owned pools, explicit close, caller-owned pool support |
+| Transactional publication | Regular enqueue | Caller-owned transaction API: commit business data and tasks together |
+| Coordination | No PostgreSQL rate-limit/barrier backend | PostgreSQL rate limits, durable barriers and standard GroupCallbacks without Redis |
+| Task operations | stats, purge, recover, flush | Failed-task inspection/targeted retry, pause/resume, cooperative cancellation and opt-in attempt history |
+| Publication controls | Single-message enqueue | Deduplication keys/TTL and atomic batches of up to 1000; deduplication is not exactly-once execution |
+| Scheduling | Per-message delay | Also a multi-process fixed-interval scheduler; no cron/calendar expressions |
+| Observability | Basic CLI statistics | Ready/scheduled backlog, task age and optional Prometheus collector |
+| Storage namespaces | Configurable schema/table prefix; shared channel/lock domains | Instance-specific queries and schema/prefix-specific channels/locks; storage separation is not access control |
+| Dramatiq integration | Broker/results implementation | Verified pipelines, groups, AsyncIO and standard middleware; these are Dramatiq features, not a replacement orchestration engine |
+| Compatibility evidence | Historical upstream tests | CI on Python 3.10/3.13/3.14 × PostgreSQL 14/18, functional tests and installed-wheel checks |
+
+See [migration from dramatiq-pg](docs/migration.md) for dependency, pool,
+import, CLI and database changes, including backup and rollback. No throughput
+advantage is claimed here. The 0.13.0rc3 prerelease is available on PyPI; the legacy Django
+integration is archived and has not been verified with IDDQueue.
+
+## Development and support
+
+Use [CONTRIBUTING](CONTRIBUTING.md) for locked uv setup and the full release gate.
+Functional tests require a dedicated PostgreSQL instance: they terminate sessions
+and crash/restart workers. Project planning and evidence live in
+[OpenSpec](openspec/roadmap.md). Report reproducible issues on
+[GitHub](https://github.com/wa-pis/iddqueue/issues).
+
+## License and credits
+
+Released under the [PostgreSQL license](LICENSE), preserving
+`Copyright (c) 2019, DALIBO` and the original author Étienne BERSAC.
+
+Thanks to upstream contributors Andy Freeland, Curtis Maloney (Django support),
+Federico Caselli, Giuseppe Papallo and Rafal Kwasny. The historical upstream logo
+was created by [Damien CAZEILS](http://www.damiencazeils.com/).
+[Original changelog](docs/changelog.rst) remains available as historical context.

iddqueue-0.13.0rc3/README.md

@@ -0,0 +1,162 @@
+# IDDQueue
+
+[Documentation](https://wa-pis.github.io/iddqueue/) · [Step-by-step setup](https://wa-pis.github.io/iddqueue/walkthrough/)
+
+A PostgreSQL broker and Results backend for [Dramatiq](https://dramatiq.io/),
+built on synchronous Psycopg 3. Tasks, results and coordination live in PostgreSQL;
+no Redis or ORM is required by IDDQueue.
+
+**Current release: [0.13.0rc3](https://pypi.org/project/iddqueue/0.13.0rc3/)** —
+a release candidate for evaluation. [GitHub assets](https://github.com/wa-pis/iddqueue/releases/tag/v0.13.0rc3)
+include the wheel, sdist and SHA256 checksums.
+
+IDDQueue is a fork of [DALIBO's dramatiq-pg](https://gitlab.com/dalibo/dramatiq-pg)
+([original package](https://pypi.org/project/dramatiq-pg/)). It preserves the
+PostgreSQL license and original contributor credits.
+
+## What it provides
+
+- Durable JSONB task/results storage, delayed tasks and crash recovery.
+- Transactional publication alongside business writes; atomic batches and deduplication.
+- PostgreSQL rate limiters, barriers and Dramatiq group callbacks.
+- Pause/resume, cooperative cancellation, failed-task inspection/retry and attempt history.
+- Fixed-interval scheduling and optional Prometheus queue metrics.
+- Schema/prefix isolation and a [tested FastAPI example](docs/fastapi.md).
+
+Delivery is **at least once**: actors and callbacks must be idempotent.
+LISTEN/NOTIFY carries message IDs as wakeup hints; consumers claim authoritative
+SQL rows with session advisory locks. See [behavior and limits](docs/user-guide.md).
+
+## Install
+
+Requires Python 3.10+, Dramatiq 2.2.1+ and Psycopg 3.3.6+.
+CI tests Python **3.10 / 3.13 / 3.14** with PostgreSQL **14 / 18**.
+See [compatibility and API stability](SUPPORT.md).
+
+```sh
+uv venv
+uv pip install "iddqueue[binary]==0.13.0rc3"
+```
+
+The `binary` extra supplies libpq. Base installs require system libpq;
+`[binary,monitoring]` also enables Prometheus support.
+
+**Migrating from dramatiq-pg?** Follow the [migration guide](docs/migration.md)
+before changing imports, pools or database storage. Stop all participants,
+back up data, run the upgrade and restart consistently. Fresh initialization
+and an existing-storage upgrade are different operations.
+
+## Quickstart
+
+Configure PostgreSQL using `PGHOST`, `PGPORT`, `PGUSER`, `PGDATABASE` and your
+normal credential mechanism. Initialize fresh storage:
+
+```sh
+uv run --no-sync iddqueue init
+```
+
+For existing storage, use `iddqueue upgrade` after following the migration guide.
+Both commands manage the complete feature schema.
+
+Save as `tasks.py`:
+
+```python
+import dramatiq
+from iddqueue import PostgresBroker
+
+broker = PostgresBroker()
+dramatiq.set_broker(broker)
+
+@dramatiq.actor(broker=broker, store_results=True)
+def add(left: int, right: int):
+    return left + right
+```
+
+Start a separate worker from the same directory/environment:
+
+```sh
+uv run --no-sync dramatiq --use-spawn --processes=1 --threads=1 tasks
+```
+
+In another terminal, publish and read the result:
+
+```sh
+uv run --no-sync python - <<'PY'
+from tasks import add, broker
+
+try:
+    message = add.send(2, 3)
+    print(message.message_id)
+    print(message.get_result(backend=broker.backend, block=True, timeout=10000))
+finally:
+    broker.close()
+PY
+```
+
+The result is `5`; timeouts are milliseconds. Each process creates its own pool.
+Close broker-owned pools on shutdown; callers close pools they supply.
+
+## Guides and examples
+
+[Domain actors and one worker bootstrap](docs/domains.md) are included since RC3.
+
+- [Documentation index](docs/index.md), [user guide](docs/user-guide.md), [API](docs/api.md).
+- [Detailed recipes](docs/recipes.md): transactions, middleware, controls, metrics and schedules.
+- [FastAPI](docs/fastapi.md): lifespan, async endpoint thread offload and separate worker.
+- [Deployment and retention](docs/deployment-guide.md), [why PostgreSQL](docs/why.md).
+- [RC notes](docs/rc-0.13.0rc3.md), [changelog](CHANGELOG.md), [release checks](docs/release.md).
+
+The broker and result APIs are synchronous. Async actors use Dramatiq's AsyncIO
+middleware; FastAPI async endpoints offload publication to a thread.
+Transactional enqueue does not accept a Psycopg AsyncConnection.
+
+Session-bound locks/listeners require persistent database sessions; PgBouncer
+transaction pooling is unsuitable. Namespace separation is not access control.
+The legacy [django-dramatiq-pg](https://github.com/uptick/django-dramatiq-pg/)
+integration has been archived since September 3, 2024; IDDQueue compatibility
+is unverified. No tested Django integration is currently provided.
+
+## Compared with dramatiq-pg
+
+Baseline: [dramatiq-pg 0.12.0](https://pypi.org/project/dramatiq-pg/0.12.0/),
+compared with the published IDDQueue 0.13.0rc3 prerelease. This describes the published version, not every
+future upstream revision. Both projects provide a PostgreSQL Dramatiq broker.
+
+| Area | dramatiq-pg 0.12.0 | IDDQueue 0.13.0rc3 / practical benefit |
+| --- | --- | --- |
+| Core storage and delivery | JSONB tasks/results, delayed tasks, LISTEN/NOTIFY, advisory locks, recovery and maintenance CLI | Preserved; at-least-once delivery still requires idempotent actors |
+| Actor initialization | Broker configured before actor imports | Opt-in Domain declarations import before DSN; explicit startup registration and domain queues |
+| Runtime | Python >=3.6,<4; Dramatiq >=1.5,<2; Psycopg 2 | Python >=3.10,<4; Dramatiq >=2.2.1,<3; synchronous Psycopg 3 and psycopg-pool |
+| Pool lifecycle | Psycopg 2 pools | Lazy owned pools, explicit close, caller-owned pool support |
+| Transactional publication | Regular enqueue | Caller-owned transaction API: commit business data and tasks together |
+| Coordination | No PostgreSQL rate-limit/barrier backend | PostgreSQL rate limits, durable barriers and standard GroupCallbacks without Redis |
+| Task operations | stats, purge, recover, flush | Failed-task inspection/targeted retry, pause/resume, cooperative cancellation and opt-in attempt history |
+| Publication controls | Single-message enqueue | Deduplication keys/TTL and atomic batches of up to 1000; deduplication is not exactly-once execution |
+| Scheduling | Per-message delay | Also a multi-process fixed-interval scheduler; no cron/calendar expressions |
+| Observability | Basic CLI statistics | Ready/scheduled backlog, task age and optional Prometheus collector |
+| Storage namespaces | Configurable schema/table prefix; shared channel/lock domains | Instance-specific queries and schema/prefix-specific channels/locks; storage separation is not access control |
+| Dramatiq integration | Broker/results implementation | Verified pipelines, groups, AsyncIO and standard middleware; these are Dramatiq features, not a replacement orchestration engine |
+| Compatibility evidence | Historical upstream tests | CI on Python 3.10/3.13/3.14 × PostgreSQL 14/18, functional tests and installed-wheel checks |
+
+See [migration from dramatiq-pg](docs/migration.md) for dependency, pool,
+import, CLI and database changes, including backup and rollback. No throughput
+advantage is claimed here. The 0.13.0rc3 prerelease is available on PyPI; the legacy Django
+integration is archived and has not been verified with IDDQueue.
+
+## Development and support
+
+Use [CONTRIBUTING](CONTRIBUTING.md) for locked uv setup and the full release gate.
+Functional tests require a dedicated PostgreSQL instance: they terminate sessions
+and crash/restart workers. Project planning and evidence live in
+[OpenSpec](openspec/roadmap.md). Report reproducible issues on
+[GitHub](https://github.com/wa-pis/iddqueue/issues).
+
+## License and credits
+
+Released under the [PostgreSQL license](LICENSE), preserving
+`Copyright (c) 2019, DALIBO` and the original author Étienne BERSAC.
+
+Thanks to upstream contributors Andy Freeland, Curtis Maloney (Django support),
+Federico Caselli, Giuseppe Papallo and Rafal Kwasny. The historical upstream logo
+was created by [Damien CAZEILS](http://www.damiencazeils.com/).
+[Original changelog](docs/changelog.rst) remains available as historical context.

{iddqueue-0.13.0rc1 → iddqueue-0.13.0rc3}/iddqueue/__init__.py

@@ -1,10 +1,12 @@
 from .broker import PostgresBroker
 from .cancellation import ResultCancelled
+from .domains import Domain
 from .rate_limits import PostgresRateLimiterBackend
 from .results import PostgresBackend
 from .schema import generate_coordination_sql, generate_init_sql, generate_upgrade_sql
 
 __all__ = [
+    "Domain",
     "ResultCancelled",
     "PostgresBackend",
     "PostgresBroker",

{iddqueue-0.13.0rc1 → iddqueue-0.13.0rc3}/iddqueue/cli.py

@@ -114,7 +114,6 @@ def make_argument_parser():
         "-d",
         "--dsn",
         "--connstring",
-        action="store",
         dest="url",
         default="",
         metavar="CONNSTRING",
@@ -122,8 +121,6 @@ def make_argument_parser():
     )
     parser.add_argument(
         "--schemaname",
-        action="store",
-        dest="schemaname",
         default="dramatiq",
         metavar="SCHEMA",
         help=(
@@ -132,8 +129,6 @@ def make_argument_parser():
     )
     parser.add_argument(
         "--prefix",
-        action="store",
-        dest="prefix",
         default="",
         metavar="PREFIX",
         help='Prefix for table name for message. Default is "%(default)s".',

iddqueue-0.13.0rc3/iddqueue/domains.py

@@ -0,0 +1,70 @@
+"""Framework-independent declarations, bound explicitly before worker startup."""
+
+import re
+
+from dramatiq import Actor
+from dramatiq.broker import Broker
+
+
+class _DeclarationBroker(Broker):
+    def __init__(self):
+        super().__init__(middleware=[])
+
+    def declare_queue(self, queue_name):
+        # Declarations create no transport resources.
+        pass
+
+    def enqueue(self, message, *, delay=None):
+        raise RuntimeError("Register the actor's Domain with a broker before sending")
+
+
+class Domain:
+    """One queue and qualified actor names; register once per process."""
+
+    def __init__(self, name):
+        if not isinstance(name, str) or not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9._-]*", name):
+            raise ValueError("domain name must be a valid Dramatiq queue name")
+        self.name = name
+        self._declarations = _DeclarationBroker()
+        self._broker = None
+        self._failed = False
+
+    def actor(self, fn=None, *, actor_name=None, priority=0, **options):
+        def decorate(fn):
+            if self._broker is not None or self._failed:
+                raise RuntimeError("Declare all domain actors before registration")
+            if "broker" in options or "queue_name" in options:
+                raise ValueError("Domain owns the broker and queue_name")
+            name = actor_name if actor_name is not None else fn.__name__
+            if not isinstance(name, str) or not name:
+                raise ValueError("actor_name must be a nonempty string")
+            return Actor(fn, broker=self._declarations, actor_name=f"{self.name}.{name}",
+                         queue_name=self.name, priority=priority, options=dict(options))
+        return decorate if fn is None else decorate(fn)
+
+    def register(self, broker):
+        if self._failed:
+            raise RuntimeError("Registration failed; create a new Domain and broker")
+        if not isinstance(broker, Broker):
+            raise TypeError("register requires a Dramatiq Broker")
+        if broker is self._broker:
+            return
+        if self._broker is not None:
+            raise RuntimeError("Domain is already registered with another broker")
+        actors = list(self._declarations.actors.values())
+        for actor in actors:
+            invalid = set(actor.options) - broker.actor_options
+            if invalid:
+                raise ValueError(f"Undefined actor options for {actor.actor_name}: {sorted(invalid)}")
+            if actor.actor_name in broker.actors:
+                raise ValueError(f"Actor {actor.actor_name!r} is already registered")
+        try:
+            for actor in actors:
+                actor.broker = broker
+                broker.declare_actor(actor)
+        except Exception:
+            for actor in actors:
+                actor.broker = self._declarations
+            self._failed = True
+            raise
+        self._broker = broker

{iddqueue-0.13.0rc1 → iddqueue-0.13.0rc3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "iddqueue"
-version = "0.13.0rc1"
+version = "0.13.0rc3"
 description = "Postgres Broker for Dramatiq Task Queue"
 authors = [{name = "Étienne BERSAC"}]
 license = "PostgreSQL"
@@ -29,12 +29,14 @@ iddqueue = "iddqueue.cli:entrypoint"
 [dependency-groups]
 dev = [
     "pytest>=8", "pytest-timeout>=2", "pytest-mock>=3",
-    "dramatiq[watch]>=2.2.1,<3", "psycopg[binary]>=3.3.6,<4",
+    "psycopg[binary]>=3.3.6,<4",
     "docutils>=0.21", "pygments>=2.20", "ruff>=0.9",
 ]
 
 fastapi-example = ["fastapi>=0.115,<1", "httpx>=0.28,<1", "uvicorn>=0.34,<1"]
 
+docs = ["mkdocs>=1.6.1,<2"]
+
 [build-system]
 requires = ["uv_build>=0.11.23,<0.12"]
 build-backend = "uv_build"