hookrelay 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hookrelay-0.2.0/.github/workflows/ci.yml +50 -0
- hookrelay-0.2.0/.github/workflows/release.yml +36 -0
- hookrelay-0.2.0/.gitignore +16 -0
- hookrelay-0.2.0/LICENSE +21 -0
- hookrelay-0.2.0/PKG-INFO +210 -0
- hookrelay-0.2.0/README.md +161 -0
- hookrelay-0.2.0/ROADMAP.md +43 -0
- hookrelay-0.2.0/examples/evolution_api_whatsapp/README.md +43 -0
- hookrelay-0.2.0/examples/evolution_api_whatsapp/main.py +87 -0
- hookrelay-0.2.0/pyproject.toml +73 -0
- hookrelay-0.2.0/src/hookrelay/__init__.py +19 -0
- hookrelay-0.2.0/src/hookrelay/backends/__init__.py +4 -0
- hookrelay-0.2.0/src/hookrelay/backends/base.py +74 -0
- hookrelay-0.2.0/src/hookrelay/backends/memory.py +90 -0
- hookrelay-0.2.0/src/hookrelay/backends/postgres.py +200 -0
- hookrelay-0.2.0/src/hookrelay/backends/redis.py +192 -0
- hookrelay-0.2.0/src/hookrelay/backends/sqlite.py +223 -0
- hookrelay-0.2.0/src/hookrelay/cli.py +122 -0
- hookrelay-0.2.0/src/hookrelay/exceptions.py +10 -0
- hookrelay-0.2.0/src/hookrelay/fastapi.py +56 -0
- hookrelay-0.2.0/src/hookrelay/metrics.py +79 -0
- hookrelay-0.2.0/src/hookrelay/models.py +48 -0
- hookrelay-0.2.0/src/hookrelay/retry.py +37 -0
- hookrelay-0.2.0/src/hookrelay/worker.py +89 -0
- hookrelay-0.2.0/tests/conftest.py +129 -0
- hookrelay-0.2.0/tests/test_backends.py +107 -0
- hookrelay-0.2.0/tests/test_cli.py +103 -0
- hookrelay-0.2.0/tests/test_fastapi_integration.py +82 -0
- hookrelay-0.2.0/tests/test_metrics.py +116 -0
- hookrelay-0.2.0/tests/test_purge.py +75 -0
- hookrelay-0.2.0/tests/test_redis_lease.py +76 -0
- hookrelay-0.2.0/tests/test_retry.py +36 -0
- hookrelay-0.2.0/tests/test_worker.py +114 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
services:
|
|
12
|
+
postgres:
|
|
13
|
+
image: postgres:16
|
|
14
|
+
env:
|
|
15
|
+
POSTGRES_USER: hookrelay
|
|
16
|
+
POSTGRES_PASSWORD: hookrelay
|
|
17
|
+
POSTGRES_DB: hookrelay_test
|
|
18
|
+
ports: ["5432:5432"]
|
|
19
|
+
options: >-
|
|
20
|
+
--health-cmd pg_isready
|
|
21
|
+
--health-interval 10s
|
|
22
|
+
--health-timeout 5s
|
|
23
|
+
--health-retries 5
|
|
24
|
+
redis:
|
|
25
|
+
image: redis:7
|
|
26
|
+
ports: ["6379:6379"]
|
|
27
|
+
options: >-
|
|
28
|
+
--health-cmd "redis-cli ping"
|
|
29
|
+
--health-interval 10s
|
|
30
|
+
--health-timeout 5s
|
|
31
|
+
--health-retries 5
|
|
32
|
+
strategy:
|
|
33
|
+
matrix:
|
|
34
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
35
|
+
env:
|
|
36
|
+
HOOKRELAY_TEST_DATABASE_URL: postgresql+asyncpg://hookrelay:hookrelay@localhost:5432/hookrelay_test
|
|
37
|
+
HOOKRELAY_TEST_REDIS_URL: redis://localhost:6379/0
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v4
|
|
40
|
+
- uses: actions/setup-python@v5
|
|
41
|
+
with:
|
|
42
|
+
python-version: ${{ matrix.python-version }}
|
|
43
|
+
- name: Install dependencies
|
|
44
|
+
run: pip install -e ".[dev]"
|
|
45
|
+
- name: Lint
|
|
46
|
+
run: ruff check src tests
|
|
47
|
+
- name: Type check
|
|
48
|
+
run: mypy src
|
|
49
|
+
- name: Test
|
|
50
|
+
run: pytest -v
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
- name: Build sdist and wheel
|
|
16
|
+
run: |
|
|
17
|
+
python -m pip install --upgrade build
|
|
18
|
+
python -m build
|
|
19
|
+
- uses: actions/upload-artifact@v4
|
|
20
|
+
with:
|
|
21
|
+
name: dist
|
|
22
|
+
path: dist/
|
|
23
|
+
|
|
24
|
+
publish:
|
|
25
|
+
needs: build
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
environment: pypi
|
|
28
|
+
permissions:
|
|
29
|
+
id-token: write
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/download-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
- name: Publish to PyPI
|
|
36
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
hookrelay-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Robson Carvalho
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
hookrelay-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: hookrelay
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Reliable webhook processing for Python: automatic retries with backoff, dead-letter queue and idempotency.
|
|
5
|
+
Project-URL: Homepage, https://github.com/CodeMaster-Java/hookrelay
|
|
6
|
+
Project-URL: Repository, https://github.com/CodeMaster-Java/hookrelay
|
|
7
|
+
Project-URL: Issues, https://github.com/CodeMaster-Java/hookrelay/issues
|
|
8
|
+
Author: Robson Carvalho
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: asyncio,dead-letter-queue,fastapi,reliability,retry,webhook
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Framework :: AsyncIO
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: pydantic>=2.0
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: aiosqlite>=0.19; extra == 'dev'
|
|
26
|
+
Requires-Dist: asyncpg>=0.29; extra == 'dev'
|
|
27
|
+
Requires-Dist: fastapi>=0.100; extra == 'dev'
|
|
28
|
+
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
29
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
30
|
+
Requires-Dist: prometheus-client>=0.20; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: redis>=5.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
35
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0; extra == 'dev'
|
|
36
|
+
Provides-Extra: fastapi
|
|
37
|
+
Requires-Dist: fastapi>=0.100; extra == 'fastapi'
|
|
38
|
+
Provides-Extra: metrics
|
|
39
|
+
Requires-Dist: prometheus-client>=0.20; extra == 'metrics'
|
|
40
|
+
Provides-Extra: postgres
|
|
41
|
+
Requires-Dist: asyncpg>=0.29; extra == 'postgres'
|
|
42
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0; extra == 'postgres'
|
|
43
|
+
Provides-Extra: redis
|
|
44
|
+
Requires-Dist: redis>=5.0; extra == 'redis'
|
|
45
|
+
Provides-Extra: sqlite
|
|
46
|
+
Requires-Dist: aiosqlite>=0.19; extra == 'sqlite'
|
|
47
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0; extra == 'sqlite'
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# hookrelay
|
|
51
|
+
|
|
52
|
+
[](https://github.com/CodeMaster-java/hookrelay/actions/workflows/ci.yml)
|
|
53
|
+
[](https://pypi.org/project/hookrelay/)
|
|
54
|
+
[](https://pypi.org/project/hookrelay/)
|
|
55
|
+
[](LICENSE)
|
|
56
|
+
|
|
57
|
+
Reliable webhook processing for Python: automatic retries with exponential
|
|
58
|
+
backoff, a dead-letter queue for events that never succeed, and idempotency so
|
|
59
|
+
duplicate deliveries are not processed twice.
|
|
60
|
+
|
|
61
|
+
Webhook providers (WhatsApp/Evolution API, Stripe, GitHub, ...) fire an HTTP
|
|
62
|
+
request and expect a fast response. If your handler is slow, flaky, or
|
|
63
|
+
depends on another service that's temporarily down, you either block the
|
|
64
|
+
request or silently drop the event. hookrelay separates *receiving* a webhook
|
|
65
|
+
from *processing* it: an endpoint enqueues the event and returns immediately,
|
|
66
|
+
a background worker processes it with retries, and anything that keeps
|
|
67
|
+
failing lands in a dead-letter queue instead of disappearing.
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install hookrelay[fastapi,postgres] # or [redis] / [sqlite] instead of [postgres]
|
|
73
|
+
pip install hookrelay[metrics] # optional: Prometheus metrics for Worker
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Quickstart (FastAPI + Postgres)
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from sqlalchemy.ext.asyncio import create_async_engine
|
|
80
|
+
|
|
81
|
+
from hookrelay import RetryPolicy, Worker
|
|
82
|
+
from hookrelay.backends.postgres import PostgresBackend
|
|
83
|
+
from hookrelay.fastapi import create_webhook_router
|
|
84
|
+
|
|
85
|
+
engine = create_async_engine("postgresql+asyncpg://user:pass@localhost/db")
|
|
86
|
+
backend = PostgresBackend(engine, retry_policy=RetryPolicy(max_attempts=5))
|
|
87
|
+
|
|
88
|
+
# 1. Receive: mount a router that enqueues instead of processing inline.
|
|
89
|
+
router = create_webhook_router(backend=backend, source="evolution-api")
|
|
90
|
+
app.include_router(router, prefix="/webhooks/whatsapp")
|
|
91
|
+
|
|
92
|
+
# 2. Process: run this as a background task or a separate process.
|
|
93
|
+
async def handle(event):
|
|
94
|
+
... # your business logic; raise to trigger a retry
|
|
95
|
+
|
|
96
|
+
worker = Worker(backend, handle)
|
|
97
|
+
await worker.run_forever()
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
On startup, create the table once with `await backend.init_schema()`, or
|
|
101
|
+
generate a migration from `hookrelay.backends.postgres.metadata` if your
|
|
102
|
+
project manages schema through Alembic.
|
|
103
|
+
|
|
104
|
+
## How it works
|
|
105
|
+
|
|
106
|
+
- **Backend**: stores events and decides what's due for (re)processing.
|
|
107
|
+
`PostgresBackend` and `RedisBackend` are safe for concurrent workers;
|
|
108
|
+
`SQLiteBackend` is safe within a single process; `MemoryBackend` is for
|
|
109
|
+
local development and tests.
|
|
110
|
+
- **Worker**: polls the backend, calls your handler, and acks or fails the
|
|
111
|
+
event based on whether the handler raised. `Worker(..., concurrency=N)`
|
|
112
|
+
runs up to `N` handlers from the same claimed batch at once, bounded by an
|
|
113
|
+
`asyncio.Semaphore`; each event is still individually acked or failed, so
|
|
114
|
+
the retry and dead-letter contract doesn't change.
|
|
115
|
+
- **RetryPolicy**: exponential backoff with jitter (`base_delay * multiplier
|
|
116
|
+
** attempt`, capped at `max_delay`), shared by every backend so retry
|
|
117
|
+
behavior doesn't depend on which one you picked.
|
|
118
|
+
- **Idempotency**: pass `idempotency_key` when building a `WebhookEvent` (or
|
|
119
|
+
an `idempotency_key` extractor to `create_webhook_router`) and a second
|
|
120
|
+
delivery with the same key is a no-op.
|
|
121
|
+
- **Dead-letter queue**: once `max_attempts` is exhausted, an event moves to
|
|
122
|
+
`EventStatus.DEAD_LETTER`. Inspect it with `backend.list_dead_letters()`
|
|
123
|
+
and retry it manually with `backend.requeue_dead_letter(event_id)`, or
|
|
124
|
+
from the command line with `hookrelay dead-letters list|requeue` (see
|
|
125
|
+
below).
|
|
126
|
+
- **Metrics**: pass a `WorkerMetrics` implementation to `Worker(...,
|
|
127
|
+
metrics=...)` for counters of processed, retried, and dead-lettered events
|
|
128
|
+
plus a handler-duration histogram. `hookrelay.metrics.PrometheusMetrics`
|
|
129
|
+
is a ready-made one (requires the `metrics` extra); implement the
|
|
130
|
+
`WorkerMetrics` protocol yourself to plug into anything else.
|
|
131
|
+
|
|
132
|
+
## Choosing a backend
|
|
133
|
+
|
|
134
|
+
| | Postgres | Redis | SQLite | Memory |
|
|
135
|
+
|---|---|---|---|---|
|
|
136
|
+
| Multiple workers | Yes (`SELECT ... FOR UPDATE SKIP LOCKED`) | Yes, for modest concurrency | No (single process only) | No |
|
|
137
|
+
| Recovers a worker that crashes mid-processing | Yes | Yes, call `reap_stale_claims()` periodically | No | No |
|
|
138
|
+
| Extra infra required | Postgres (you probably already have it) | Redis | None (a local file) | None |
|
|
139
|
+
|
|
140
|
+
If you're unsure, start with Postgres: you likely already run one, and it
|
|
141
|
+
gives you the strongest guarantees. `SQLiteBackend` is a good fit for a
|
|
142
|
+
single-process deployment or script where standing up Postgres or Redis just
|
|
143
|
+
for retry bookkeeping isn't worth it.
|
|
144
|
+
|
|
145
|
+
## Maintenance
|
|
146
|
+
|
|
147
|
+
Two housekeeping operations are opt-in: hookrelay never runs them on its own,
|
|
148
|
+
so wire them into whatever periodic task runner you already use (a cron job,
|
|
149
|
+
an `asyncio` task, ...).
|
|
150
|
+
|
|
151
|
+
- **`RedisBackend.reap_stale_claims()`**: a claimed event carries a lease
|
|
152
|
+
(`claim_lease_seconds`, default 300). If the worker that claimed it
|
|
153
|
+
crashes before acking or failing it, the event is stuck until you call
|
|
154
|
+
`reap_stale_claims()`, which requeues it (or dead-letters it, if that was
|
|
155
|
+
its last attempt) exactly like any other failure. Call it every
|
|
156
|
+
`claim_lease_seconds / 2` or so.
|
|
157
|
+
- **`PostgresBackend.purge()` / `SQLiteBackend.purge()`**: successful and
|
|
158
|
+
dead-lettered events stay in the table indefinitely otherwise. Call
|
|
159
|
+
`backend.purge(older_than=some_datetime)` periodically to delete them past
|
|
160
|
+
a retention window you choose.
|
|
161
|
+
|
|
162
|
+
## CLI
|
|
163
|
+
|
|
164
|
+
Installing hookrelay also installs a `hookrelay` command for inspecting and
|
|
165
|
+
recovering dead-lettered events against a running deployment:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
hookrelay dead-letters list --backend postgresql+asyncpg://user:pass@localhost/db
|
|
169
|
+
hookrelay dead-letters requeue <event-id> --backend redis://localhost:6379/0
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
`--backend` accepts a Postgres, Redis, or SQLite URL; the CLI picks the
|
|
173
|
+
matching backend implementation from its scheme and only needs that
|
|
174
|
+
backend's extra installed.
|
|
175
|
+
|
|
176
|
+
## Known limitations
|
|
177
|
+
|
|
178
|
+
- There's no built-in dashboard for dead-letter events; `list_dead_letters()`,
|
|
179
|
+
`requeue_dead_letter()`, and the `hookrelay dead-letters` CLI are meant to
|
|
180
|
+
be wired into your own admin tooling, not to replace it.
|
|
181
|
+
- `SQLiteBackend` is safe for concurrent calls within one process (guarded
|
|
182
|
+
by an internal lock) but not for multiple worker processes writing to the
|
|
183
|
+
same database file; use `PostgresBackend` if you need that.
|
|
184
|
+
|
|
185
|
+
## Roadmap
|
|
186
|
+
|
|
187
|
+
Planned improvements and deliberate non-goals live in [ROADMAP.md](ROADMAP.md).
|
|
188
|
+
|
|
189
|
+
## Example
|
|
190
|
+
|
|
191
|
+
See [`examples/evolution_api_whatsapp`](examples/evolution_api_whatsapp) for a
|
|
192
|
+
complete FastAPI app that receives Evolution API (self-hosted WhatsApp)
|
|
193
|
+
webhooks with signature verification and idempotency.
|
|
194
|
+
|
|
195
|
+
## Contributing
|
|
196
|
+
|
|
197
|
+
Issues and PRs are welcome. Run the test suite with:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
pip install -e ".[dev]"
|
|
201
|
+
pytest
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Postgres- and Redis-backed tests are skipped automatically unless
|
|
205
|
+
`HOOKRELAY_TEST_DATABASE_URL` and `HOOKRELAY_TEST_REDIS_URL` are set; see
|
|
206
|
+
`.github/workflows/ci.yml` for how CI provides both.
|
|
207
|
+
|
|
208
|
+
## License
|
|
209
|
+
|
|
210
|
+
MIT
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# hookrelay
|
|
2
|
+
|
|
3
|
+
[](https://github.com/CodeMaster-java/hookrelay/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/hookrelay/)
|
|
5
|
+
[](https://pypi.org/project/hookrelay/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
Reliable webhook processing for Python: automatic retries with exponential
|
|
9
|
+
backoff, a dead-letter queue for events that never succeed, and idempotency so
|
|
10
|
+
duplicate deliveries are not processed twice.
|
|
11
|
+
|
|
12
|
+
Webhook providers (WhatsApp/Evolution API, Stripe, GitHub, ...) fire an HTTP
|
|
13
|
+
request and expect a fast response. If your handler is slow, flaky, or
|
|
14
|
+
depends on another service that's temporarily down, you either block the
|
|
15
|
+
request or silently drop the event. hookrelay separates *receiving* a webhook
|
|
16
|
+
from *processing* it: an endpoint enqueues the event and returns immediately,
|
|
17
|
+
a background worker processes it with retries, and anything that keeps
|
|
18
|
+
failing lands in a dead-letter queue instead of disappearing.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install hookrelay[fastapi,postgres] # or [redis] / [sqlite] instead of [postgres]
|
|
24
|
+
pip install hookrelay[metrics] # optional: Prometheus metrics for Worker
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Quickstart (FastAPI + Postgres)
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from sqlalchemy.ext.asyncio import create_async_engine
|
|
31
|
+
|
|
32
|
+
from hookrelay import RetryPolicy, Worker
|
|
33
|
+
from hookrelay.backends.postgres import PostgresBackend
|
|
34
|
+
from hookrelay.fastapi import create_webhook_router
|
|
35
|
+
|
|
36
|
+
engine = create_async_engine("postgresql+asyncpg://user:pass@localhost/db")
|
|
37
|
+
backend = PostgresBackend(engine, retry_policy=RetryPolicy(max_attempts=5))
|
|
38
|
+
|
|
39
|
+
# 1. Receive: mount a router that enqueues instead of processing inline.
|
|
40
|
+
router = create_webhook_router(backend=backend, source="evolution-api")
|
|
41
|
+
app.include_router(router, prefix="/webhooks/whatsapp")
|
|
42
|
+
|
|
43
|
+
# 2. Process: run this as a background task or a separate process.
|
|
44
|
+
async def handle(event):
|
|
45
|
+
... # your business logic; raise to trigger a retry
|
|
46
|
+
|
|
47
|
+
worker = Worker(backend, handle)
|
|
48
|
+
await worker.run_forever()
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
On startup, create the table once with `await backend.init_schema()`, or
|
|
52
|
+
generate a migration from `hookrelay.backends.postgres.metadata` if your
|
|
53
|
+
project manages schema through Alembic.
|
|
54
|
+
|
|
55
|
+
## How it works
|
|
56
|
+
|
|
57
|
+
- **Backend**: stores events and decides what's due for (re)processing.
|
|
58
|
+
`PostgresBackend` and `RedisBackend` are safe for concurrent workers;
|
|
59
|
+
`SQLiteBackend` is safe within a single process; `MemoryBackend` is for
|
|
60
|
+
local development and tests.
|
|
61
|
+
- **Worker**: polls the backend, calls your handler, and acks or fails the
|
|
62
|
+
event based on whether the handler raised. `Worker(..., concurrency=N)`
|
|
63
|
+
runs up to `N` handlers from the same claimed batch at once, bounded by an
|
|
64
|
+
`asyncio.Semaphore`; each event is still individually acked or failed, so
|
|
65
|
+
the retry and dead-letter contract doesn't change.
|
|
66
|
+
- **RetryPolicy**: exponential backoff with jitter (`base_delay * multiplier
|
|
67
|
+
** attempt`, capped at `max_delay`), shared by every backend so retry
|
|
68
|
+
behavior doesn't depend on which one you picked.
|
|
69
|
+
- **Idempotency**: pass `idempotency_key` when building a `WebhookEvent` (or
|
|
70
|
+
an `idempotency_key` extractor to `create_webhook_router`) and a second
|
|
71
|
+
delivery with the same key is a no-op.
|
|
72
|
+
- **Dead-letter queue**: once `max_attempts` is exhausted, an event moves to
|
|
73
|
+
`EventStatus.DEAD_LETTER`. Inspect it with `backend.list_dead_letters()`
|
|
74
|
+
and retry it manually with `backend.requeue_dead_letter(event_id)`, or
|
|
75
|
+
from the command line with `hookrelay dead-letters list|requeue` (see
|
|
76
|
+
below).
|
|
77
|
+
- **Metrics**: pass a `WorkerMetrics` implementation to `Worker(...,
|
|
78
|
+
metrics=...)` for counters of processed, retried, and dead-lettered events
|
|
79
|
+
plus a handler-duration histogram. `hookrelay.metrics.PrometheusMetrics`
|
|
80
|
+
is a ready-made one (requires the `metrics` extra); implement the
|
|
81
|
+
`WorkerMetrics` protocol yourself to plug into anything else.
|
|
82
|
+
|
|
83
|
+
## Choosing a backend
|
|
84
|
+
|
|
85
|
+
| | Postgres | Redis | SQLite | Memory |
|
|
86
|
+
|---|---|---|---|---|
|
|
87
|
+
| Multiple workers | Yes (`SELECT ... FOR UPDATE SKIP LOCKED`) | Yes, for modest concurrency | No (single process only) | No |
|
|
88
|
+
| Recovers a worker that crashes mid-processing | Yes | Yes, call `reap_stale_claims()` periodically | No | No |
|
|
89
|
+
| Extra infra required | Postgres (you probably already have it) | Redis | None (a local file) | None |
|
|
90
|
+
|
|
91
|
+
If you're unsure, start with Postgres: you likely already run one, and it
|
|
92
|
+
gives you the strongest guarantees. `SQLiteBackend` is a good fit for a
|
|
93
|
+
single-process deployment or script where standing up Postgres or Redis just
|
|
94
|
+
for retry bookkeeping isn't worth it.
|
|
95
|
+
|
|
96
|
+
## Maintenance
|
|
97
|
+
|
|
98
|
+
Two housekeeping operations are opt-in: hookrelay never runs them on its own,
|
|
99
|
+
so wire them into whatever periodic task runner you already use (a cron job,
|
|
100
|
+
an `asyncio` task, ...).
|
|
101
|
+
|
|
102
|
+
- **`RedisBackend.reap_stale_claims()`**: a claimed event carries a lease
|
|
103
|
+
(`claim_lease_seconds`, default 300). If the worker that claimed it
|
|
104
|
+
crashes before acking or failing it, the event is stuck until you call
|
|
105
|
+
`reap_stale_claims()`, which requeues it (or dead-letters it, if that was
|
|
106
|
+
its last attempt) exactly like any other failure. Call it every
|
|
107
|
+
`claim_lease_seconds / 2` or so.
|
|
108
|
+
- **`PostgresBackend.purge()` / `SQLiteBackend.purge()`**: successful and
|
|
109
|
+
dead-lettered events stay in the table indefinitely otherwise. Call
|
|
110
|
+
`backend.purge(older_than=some_datetime)` periodically to delete them past
|
|
111
|
+
a retention window you choose.
|
|
112
|
+
|
|
113
|
+
## CLI
|
|
114
|
+
|
|
115
|
+
Installing hookrelay also installs a `hookrelay` command for inspecting and
|
|
116
|
+
recovering dead-lettered events against a running deployment:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
hookrelay dead-letters list --backend postgresql+asyncpg://user:pass@localhost/db
|
|
120
|
+
hookrelay dead-letters requeue <event-id> --backend redis://localhost:6379/0
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`--backend` accepts a Postgres, Redis, or SQLite URL; the CLI picks the
|
|
124
|
+
matching backend implementation from its scheme and only needs that
|
|
125
|
+
backend's extra installed.
|
|
126
|
+
|
|
127
|
+
## Known limitations
|
|
128
|
+
|
|
129
|
+
- There's no built-in dashboard for dead-letter events; `list_dead_letters()`,
|
|
130
|
+
`requeue_dead_letter()`, and the `hookrelay dead-letters` CLI are meant to
|
|
131
|
+
be wired into your own admin tooling, not to replace it.
|
|
132
|
+
- `SQLiteBackend` is safe for concurrent calls within one process (guarded
|
|
133
|
+
by an internal lock) but not for multiple worker processes writing to the
|
|
134
|
+
same database file; use `PostgresBackend` if you need that.
|
|
135
|
+
|
|
136
|
+
## Roadmap
|
|
137
|
+
|
|
138
|
+
Planned improvements and deliberate non-goals live in [ROADMAP.md](ROADMAP.md).
|
|
139
|
+
|
|
140
|
+
## Example
|
|
141
|
+
|
|
142
|
+
See [`examples/evolution_api_whatsapp`](examples/evolution_api_whatsapp) for a
|
|
143
|
+
complete FastAPI app that receives Evolution API (self-hosted WhatsApp)
|
|
144
|
+
webhooks with signature verification and idempotency.
|
|
145
|
+
|
|
146
|
+
## Contributing
|
|
147
|
+
|
|
148
|
+
Issues and PRs are welcome. Run the test suite with:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
pip install -e ".[dev]"
|
|
152
|
+
pytest
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Postgres- and Redis-backed tests are skipped automatically unless
|
|
156
|
+
`HOOKRELAY_TEST_DATABASE_URL` and `HOOKRELAY_TEST_REDIS_URL` are set; see
|
|
157
|
+
`.github/workflows/ci.yml` for how CI provides both.
|
|
158
|
+
|
|
159
|
+
## License
|
|
160
|
+
|
|
161
|
+
MIT
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Roadmap
|
|
2
|
+
|
|
3
|
+
Everything that was tracked here for v0.1 (stale-claim recovery for
|
|
4
|
+
`RedisBackend`, retention/cleanup for `PostgresBackend` and `SQLiteBackend`,
|
|
5
|
+
Prometheus metrics, a dead-letters CLI, concurrent handler execution, and a
|
|
6
|
+
`SQLiteBackend`) shipped in v0.2. See the README's "How it works",
|
|
7
|
+
"Maintenance", and "CLI" sections for how to use them.
|
|
8
|
+
|
|
9
|
+
## Ideas for v0.3
|
|
10
|
+
|
|
11
|
+
- **Rate limiting / throttling in `Worker`.** Nothing today bounds how fast
|
|
12
|
+
claimed events are handed to the handler; a burst of due events can drive
|
|
13
|
+
a handler straight into a downstream provider's own rate limit. A
|
|
14
|
+
configurable throttle, for example `Worker(..., max_calls_per_second=N)`,
|
|
15
|
+
following the same pattern `concurrency` already established, would let a
|
|
16
|
+
handler that calls a rate-limited API stay under it without building that
|
|
17
|
+
logic itself.
|
|
18
|
+
- **Framework adapters beyond FastAPI.**
|
|
19
|
+
`hookrelay.fastapi.create_webhook_router()` is a thin adapter over
|
|
20
|
+
`Backend.enqueue()` (verify signature, parse body, enqueue, return); the
|
|
21
|
+
same shape applies just as well to Flask and Django. Adapters for those
|
|
22
|
+
two would widen who can adopt hookrelay without touching the
|
|
23
|
+
Backend/Worker contract at all.
|
|
24
|
+
|
|
25
|
+
Nothing here is committed to a timeline; this is a place to track what's
|
|
26
|
+
deliberately left out of the current release and why, so scope stays
|
|
27
|
+
honest instead of growing by accident.
|
|
28
|
+
|
|
29
|
+
## Non-goals (for now)
|
|
30
|
+
|
|
31
|
+
- A hosted dashboard/UI. `list_dead_letters()` and `requeue_dead_letter()`
|
|
32
|
+
(and the `hookrelay dead-letters` CLI) are meant to be wired into whatever
|
|
33
|
+
admin tooling a project already has; building a UI is a much bigger
|
|
34
|
+
surface area than this library's scope.
|
|
35
|
+
- Distributed leader election or exactly-once delivery guarantees beyond
|
|
36
|
+
what each backend's native atomicity already provides. hookrelay aims for
|
|
37
|
+
at-least-once delivery with idempotency as the tool for deduplication,
|
|
38
|
+
not a distributed consensus system.
|
|
39
|
+
- Anything about the HTTP requests a handler makes, like SSRF protection or
|
|
40
|
+
HMAC-signing outgoing payloads. hookrelay only receives webhooks; `Worker`
|
|
41
|
+
calls the Python handler function you write, it never makes an HTTP
|
|
42
|
+
request on your behalf. Those concerns belong to your handler's own HTTP
|
|
43
|
+
client, not to hookrelay.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Example: reliable WhatsApp webhooks from Evolution API
|
|
2
|
+
|
|
3
|
+
Evolution API posts a JSON body like this for every event (`messages.upsert`,
|
|
4
|
+
`connection.update`, etc.):
|
|
5
|
+
|
|
6
|
+
```json
|
|
7
|
+
{
|
|
8
|
+
"event": "messages.upsert",
|
|
9
|
+
"instance": "my-instance",
|
|
10
|
+
"data": { "key": { "id": "...", "remoteJid": "..." }, "message": { "conversation": "hi" } },
|
|
11
|
+
"server_url": "https://your-evolution-api.example.com",
|
|
12
|
+
"apikey": "your-instance-token",
|
|
13
|
+
"date_time": "2026-08-26T12:00:00.000Z"
|
|
14
|
+
}
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
This example shows the pattern for making that reliable:
|
|
18
|
+
|
|
19
|
+
1. Configure the instance's webhook with a custom `Authorization: Bearer <secret>`
|
|
20
|
+
header (Evolution API's webhook settings support custom headers per instance).
|
|
21
|
+
2. `main.py` verifies that header, enqueues the event into Postgres via
|
|
22
|
+
`hookrelay`, and returns immediately.
|
|
23
|
+
3. A background `Worker` processes events, retrying with backoff on failure and
|
|
24
|
+
moving anything that exhausts its retries to the dead-letter queue.
|
|
25
|
+
4. `data.key.id` (the WhatsApp message id) is used as the idempotency key, so a
|
|
26
|
+
webhook Evolution API resends after a timeout isn't processed twice.
|
|
27
|
+
|
|
28
|
+
## Run it
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install "hookrelay[fastapi,postgres]" uvicorn
|
|
32
|
+
export WEBHOOK_SHARED_SECRET=change-me
|
|
33
|
+
export DATABASE_URL=postgresql+asyncpg://user:pass@localhost/hookrelay
|
|
34
|
+
uvicorn main:app --reload
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Point your Evolution API instance's webhook at
|
|
38
|
+
`http://your-host:8000/webhooks/whatsapp` with the header above, and inspect
|
|
39
|
+
anything that ends up dead-lettered with:
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
await backend.list_dead_letters()
|
|
43
|
+
```
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""FastAPI app that receives WhatsApp webhooks from a self-hosted Evolution API
|
|
2
|
+
instance reliably: the endpoint only enqueues, a worker processes with retries,
|
|
3
|
+
and messages that never process successfully land in the dead-letter queue
|
|
4
|
+
instead of vanishing.
|
|
5
|
+
|
|
6
|
+
Run:
|
|
7
|
+
export WEBHOOK_SHARED_SECRET=change-me
|
|
8
|
+
export DATABASE_URL=postgresql+asyncpg://user:pass@localhost/hookrelay
|
|
9
|
+
uvicorn main:app --reload
|
|
10
|
+
|
|
11
|
+
Then, in your Evolution API instance settings, point the instance's webhook at
|
|
12
|
+
this app's /webhooks/whatsapp URL and add a custom header
|
|
13
|
+
`Authorization: Bearer change-me` to the webhook configuration.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import hmac
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
from contextlib import asynccontextmanager
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from fastapi import FastAPI
|
|
26
|
+
from sqlalchemy.ext.asyncio import create_async_engine
|
|
27
|
+
|
|
28
|
+
from hookrelay import RetryPolicy, Worker
|
|
29
|
+
from hookrelay.backends.postgres import PostgresBackend
|
|
30
|
+
from hookrelay.fastapi import create_webhook_router
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger("evolution_webhook_example")
|
|
33
|
+
|
|
34
|
+
WEBHOOK_SHARED_SECRET = os.environ["WEBHOOK_SHARED_SECRET"]
|
|
35
|
+
DATABASE_URL = os.environ["DATABASE_URL"]
|
|
36
|
+
|
|
37
|
+
engine = create_async_engine(DATABASE_URL)
|
|
38
|
+
backend = PostgresBackend(engine, retry_policy=RetryPolicy(max_attempts=5, base_delay=2.0))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def verify_signature(_body: bytes, headers: Any) -> bool:
|
|
42
|
+
expected = f"Bearer {WEBHOOK_SHARED_SECRET}"
|
|
43
|
+
return hmac.compare_digest(headers.get("authorization", ""), expected)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def extract_idempotency_key(payload: dict[str, Any]) -> str | None:
|
|
47
|
+
# WhatsApp message ids are unique per message; Evolution API resends the same
|
|
48
|
+
# message.upsert event on transient delivery failures, so this dedupes retries
|
|
49
|
+
# coming from Evolution API itself, on top of hookrelay's own retry/DLQ.
|
|
50
|
+
return payload.get("data", {}).get("key", {}).get("id")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def handle_whatsapp_event(event) -> None:
|
|
54
|
+
payload = event.payload
|
|
55
|
+
event_type = payload.get("event")
|
|
56
|
+
|
|
57
|
+
if event_type == "messages.upsert":
|
|
58
|
+
message = payload["data"]
|
|
59
|
+
text = message.get("message", {}).get("conversation", "<non-text message>")
|
|
60
|
+
logger.info("WhatsApp message from %s: %s", message.get("key", {}).get("remoteJid"), text)
|
|
61
|
+
# your business logic here: persist the message, trigger a bot reply, etc.
|
|
62
|
+
# raising an exception causes hookrelay to retry this event with backoff.
|
|
63
|
+
else:
|
|
64
|
+
logger.debug("ignoring event type %s", event_type)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@asynccontextmanager
|
|
68
|
+
async def lifespan(_app: FastAPI):
|
|
69
|
+
await backend.init_schema()
|
|
70
|
+
worker = Worker(backend, handle_whatsapp_event)
|
|
71
|
+
worker_task = asyncio.create_task(worker.run_forever())
|
|
72
|
+
yield
|
|
73
|
+
worker.stop()
|
|
74
|
+
await worker_task
|
|
75
|
+
await engine.dispose()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
app = FastAPI(lifespan=lifespan)
|
|
79
|
+
app.include_router(
|
|
80
|
+
create_webhook_router(
|
|
81
|
+
backend=backend,
|
|
82
|
+
source="evolution-api",
|
|
83
|
+
verify_signature=verify_signature,
|
|
84
|
+
idempotency_key=extract_idempotency_key,
|
|
85
|
+
),
|
|
86
|
+
prefix="/webhooks/whatsapp",
|
|
87
|
+
)
|