django-qstash 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of django-qstash might be problematic. Click here for more details.
- django_qstash-0.0.4/PKG-INFO +343 -0
- django_qstash-0.0.4/README.md +314 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/pyproject.toml +4 -1
- {django_qstash-0.0.2 → django_qstash-0.0.4}/src/django_qstash/__init__.py +1 -1
- django_qstash-0.0.4/src/django_qstash/exceptions.py +25 -0
- django_qstash-0.0.4/src/django_qstash/handlers.py +143 -0
- django_qstash-0.0.4/src/django_qstash/management/commands/__init__.py +0 -0
- django_qstash-0.0.4/src/django_qstash/management/commands/clear_stale_results.py +61 -0
- django_qstash-0.0.4/src/django_qstash/results/__init__.py +0 -0
- django_qstash-0.0.4/src/django_qstash/results/admin.py +21 -0
- django_qstash-0.0.4/src/django_qstash/results/apps.py +10 -0
- django_qstash-0.0.4/src/django_qstash/results/migrations/0001_initial.py +62 -0
- django_qstash-0.0.4/src/django_qstash/results/migrations/__init__.py +0 -0
- django_qstash-0.0.4/src/django_qstash/results/models.py +36 -0
- django_qstash-0.0.4/src/django_qstash/results/services.py +33 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/src/django_qstash/tasks.py +28 -16
- django_qstash-0.0.4/src/django_qstash/views.py +21 -0
- django_qstash-0.0.4/src/django_qstash.egg-info/PKG-INFO +343 -0
- django_qstash-0.0.4/src/django_qstash.egg-info/SOURCES.txt +28 -0
- django_qstash-0.0.4/tests/test_exceptions.py +56 -0
- django_qstash-0.0.4/tests/test_handlers.py +151 -0
- django_qstash-0.0.4/tests/test_results_models.py +76 -0
- django_qstash-0.0.4/tests/test_views.py +62 -0
- django_qstash-0.0.2/PKG-INFO +0 -95
- django_qstash-0.0.2/README.md +0 -66
- django_qstash-0.0.2/src/django_qstash/views.py +0 -124
- django_qstash-0.0.2/src/django_qstash.egg-info/PKG-INFO +0 -95
- django_qstash-0.0.2/src/django_qstash.egg-info/SOURCES.txt +0 -14
- django_qstash-0.0.2/tests/test_views.py +0 -75
- {django_qstash-0.0.2 → django_qstash-0.0.4}/setup.cfg +0 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/src/django_qstash/utils.py +0 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/src/django_qstash.egg-info/dependency_links.txt +0 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/src/django_qstash.egg-info/requires.txt +0 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/src/django_qstash.egg-info/top_level.txt +0 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/tests/test_tasks.py +0 -0
- {django_qstash-0.0.2 → django_qstash-0.0.4}/tests/test_utils.py +0 -0
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: django-qstash
|
|
3
|
+
Version: 0.0.4
|
|
4
|
+
Summary: A drop-in replacement for Celery's shared_task with Upstash QStash.
|
|
5
|
+
Author-email: Justin Mitchel <justin@codingforentrepreneurs.com>
|
|
6
|
+
Project-URL: Changelog, https://github.com/jmitchel3/django-qstash
|
|
7
|
+
Project-URL: Documentation, https://github.com/jmitchel3/django-qstash
|
|
8
|
+
Project-URL: Funding, https://github.com/jmitchel3/django-qstash
|
|
9
|
+
Project-URL: Repository, https://github.com/jmitchel3/django-qstash
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Framework :: Django :: 4.2
|
|
12
|
+
Classifier: Framework :: Django :: 5.0
|
|
13
|
+
Classifier: Framework :: Django :: 5.1
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Natural Language :: English
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
Requires-Dist: django>=4.2
|
|
27
|
+
Requires-Dist: qstash>=2
|
|
28
|
+
Requires-Dist: requests>=2.30
|
|
29
|
+
|
|
30
|
+
> :warning: **BETA Software**: Working on being production-ready soon.
|
|
31
|
+
|
|
32
|
+
# django-qstash
|
|
33
|
+
|
|
34
|
+
_django-qstash_ is a drop-in replacement for Celery's `shared_task`.
|
|
35
|
+
|
|
36
|
+
To do this, we use:
|
|
37
|
+
|
|
38
|
+
- [Upstash QStash](https://upstash.com/docs/qstash/overall/getstarted)
|
|
39
|
+
- A single public _webhook_ to call `@shared_task` functions automatically
|
|
40
|
+
|
|
41
|
+
This allows us to:
|
|
42
|
+
|
|
43
|
+
- Focus just on Django
|
|
44
|
+
- Drop Celery
|
|
45
|
+
- Truly scale Django to zero
|
|
46
|
+
- Run background tasks through webhooks
|
|
47
|
+
- Cut costs
|
|
48
|
+
- Trigger GitHub Actions Workflows or GitLab CI/CD pipelines for handling other kinds of background tasks based on our project's code.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Table of Contents
|
|
52
|
+
|
|
53
|
+
- [django-qstash](#django-qstash)
|
|
54
|
+
- [Table of Contents](#table-of-contents)
|
|
55
|
+
- [Installation](#installation)
|
|
56
|
+
- [Using Pip](#using-pip)
|
|
57
|
+
- [Update Settings (`settings.py`)](#update-settings-settingspy)
|
|
58
|
+
- [Configure Webhook URL](#configure-webhook-url)
|
|
59
|
+
- [Required Environment Variables](#required-environment-variables)
|
|
60
|
+
- [Sample Project](#sample-project)
|
|
61
|
+
- [Dependencies](#dependencies)
|
|
62
|
+
- [Usage](#usage)
|
|
63
|
+
- [Define a Task](#define-a-task)
|
|
64
|
+
- [Regular Task Call](#regular-task-call)
|
|
65
|
+
- [Async Task](#async-task)
|
|
66
|
+
- [`.delay()`](#delay)
|
|
67
|
+
- [`.apply_async()`](#apply_async)
|
|
68
|
+
- [`.apply_async()` With Time Delay](#apply_async-with-time-delay)
|
|
69
|
+
- [JSON-ready Arguments](#json-ready-arguments)
|
|
70
|
+
- [Example Task](#example-task)
|
|
71
|
+
- [Configuration](#configuration)
|
|
72
|
+
- [Storing Task Results (Optional)](#storing-task-results-optional)
|
|
73
|
+
- [Clear Stale Results](#clear-stale-results)
|
|
74
|
+
- [Definitions](#definitions)
|
|
75
|
+
- [Motivation](#motivation)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
## Installation
|
|
79
|
+
|
|
80
|
+
### Using Pip
|
|
81
|
+
```bash
|
|
82
|
+
pip install django-qstash
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Update Settings (`settings.py`)
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
INSTALLED_APPS = [
|
|
89
|
+
##...
|
|
90
|
+
"django_qstash",
|
|
91
|
+
"django_qstash.results",
|
|
92
|
+
##...
|
|
93
|
+
]
|
|
94
|
+
```
|
|
95
|
+
- `django_qstash` Includes the `@shared_task` decorator and webhook view
|
|
96
|
+
- `django_qstash.results` (Optional): Store task results in Django DB
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
### Configure Webhook URL
|
|
100
|
+
|
|
101
|
+
In your `ROOT_URLCONF` (e.g. `urls.py`), add the following:
|
|
102
|
+
```python
|
|
103
|
+
from django_qstash.views import qstash_webhook_view
|
|
104
|
+
|
|
105
|
+
urlpatterns = [
|
|
106
|
+
# ...
|
|
107
|
+
path("qstash/webhook/", qstash_webhook_view),
|
|
108
|
+
# ...
|
|
109
|
+
]
|
|
110
|
+
```
|
|
111
|
+
Be sure to use this path in your `DJANGO_QSTASH_WEBHOOK_PATH` environment variable.
|
|
112
|
+
|
|
113
|
+
### Required Environment Variables
|
|
114
|
+
|
|
115
|
+
Get your QStash token and signing keys from [Upstash](https://upstash.com/).
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
QSTASH_TOKEN = "your_token"
|
|
119
|
+
QSTASH_CURRENT_SIGNING_KEY = "your_current_signing_key"
|
|
120
|
+
QSTASH_NEXT_SIGNING_KEY = "your_next_signing_key"
|
|
121
|
+
|
|
122
|
+
# required for django-qstash
|
|
123
|
+
DJANGO_QSTASH_DOMAIN = "https://example.com"
|
|
124
|
+
DJANGO_QSTASH_WEBHOOK_PATH = "/qstash/webhook/"
|
|
125
|
+
```
|
|
126
|
+
> Review [.env.sample](.env.sample) to see all the environment variables you need to set.
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
## Sample Project
|
|
130
|
+
There is a sample project in [sample_project/](sample_project/) that shows how all this is implemented.
|
|
131
|
+
|
|
132
|
+
## Dependencies
|
|
133
|
+
|
|
134
|
+
- [Python 3.10+](https://www.python.org/)
|
|
135
|
+
- [Django 5+](https://docs.djangoproject.com/)
|
|
136
|
+
- [qstash-py](https://github.com/upstash/qstash-py)
|
|
137
|
+
- [Upstash](https://upstash.com/) account
|
|
138
|
+
|
|
139
|
+
## Usage
|
|
140
|
+
|
|
141
|
+
Django-QStash revolves around the `shared_task` decorator. The goal is to be a drop-in replacement for Celery's `shared_task` decorator.
|
|
142
|
+
|
|
143
|
+
Here's how it works:
|
|
144
|
+
- Define a Task
|
|
145
|
+
- Call a Task with `.delay()` or `.apply_async()`
|
|
146
|
+
|
|
147
|
+
### Define a Task
|
|
148
|
+
```python
|
|
149
|
+
from django_qstash import shared_task
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@shared_task
|
|
153
|
+
def hello_world(name: str, age: int = None, activity: str = None):
|
|
154
|
+
if age is None:
|
|
155
|
+
print(f"Hello {name}! I see you're {activity}.")
|
|
156
|
+
return
|
|
157
|
+
print(f"Hello {name}! I see you're {activity} at {age} years old.")
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
### Regular Task Call
|
|
162
|
+
Nothing special here. Just call the function like any other to verify it works.
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
# normal function call
|
|
166
|
+
hello_world("Tony Stark", age=40, activity="building in a cave with a box of scraps.")
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Async Task
|
|
170
|
+
|
|
171
|
+
Using `.delay()` or `.apply_async()` is how you call an async task. This is modeled after Celery and it works as you'd expect.
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
#### `.delay()`
|
|
175
|
+
```python
|
|
176
|
+
hello_world.delay(
|
|
177
|
+
"Tony Stark", age=40, activity="building in a cave with a box of scraps."
|
|
178
|
+
)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
#### `.apply_async()`
|
|
182
|
+
```python
|
|
183
|
+
hello_world.apply_async(
|
|
184
|
+
args=("Tony Stark",),
|
|
185
|
+
kwargs={"activity": "building in a cave with a box of scraps."},
|
|
186
|
+
)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### `.apply_async()` With Time Delay
|
|
190
|
+
|
|
191
|
+
Just use the `countdown` parameter to delay the task by N seconds. (always in seconds): `.apply_async(*args, **kwargs, countdown=N)`
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
# async task delayed 35 seconds
|
|
196
|
+
delay_35_seconds = 35
|
|
197
|
+
hello_world.apply_async(
|
|
198
|
+
args=("Tony Stark",),
|
|
199
|
+
kwargs={"activity": "building in a cave with a box of scraps."},
|
|
200
|
+
countdown=delay_35_seconds,
|
|
201
|
+
)
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
### JSON-ready Arguments
|
|
205
|
+
|
|
206
|
+
Each argument needs to be _JSON_ serializable. The way you find out:
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
import json
|
|
210
|
+
|
|
211
|
+
data = {
|
|
212
|
+
"args": ("Tony Stark",),
|
|
213
|
+
"kwargs": {"activity": "building in a cave with a box of scraps."},
|
|
214
|
+
}
|
|
215
|
+
print(json.dumps(data))
|
|
216
|
+
# no errors, you're good to go.
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### Example Task
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
# from celery import shared_task
|
|
223
|
+
from django_qstash import shared_task
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
@shared_task
|
|
227
|
+
def math_add_task(a, b, save_to_file=False, *args, **kwargs):
|
|
228
|
+
logger.info(f"Adding {a} and {b}")
|
|
229
|
+
if save_to_file:
|
|
230
|
+
with open("math-add-result.txt", "w") as f:
|
|
231
|
+
f.write(f"{a} + {b} = {a + b}")
|
|
232
|
+
return a + b
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
Calling:
|
|
237
|
+
```python
|
|
238
|
+
math_add_task.apply_async(args=(12, 454), save_to_file=True)
|
|
239
|
+
```
|
|
240
|
+
is the same as
|
|
241
|
+
```python
|
|
242
|
+
math_add_task.delay(12, 454, save_to_file=True)
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
But if you need to delay the task, use `.apply_async()` with the `countdown` parameter.
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
five_hours = 5 * 60 * 60
|
|
249
|
+
math_add_task.apply_async(
|
|
250
|
+
args=(12, 454), kwargs={"save_to_file": True}, countdown=five_hours
|
|
251
|
+
)
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
The `.delay()` method does not support a countdown parameter because it simply passes the arguments (*args, **kwargs) to the `apply_async()` method.
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
## Configuration
|
|
258
|
+
|
|
259
|
+
In Django settings, you can configure the following:
|
|
260
|
+
|
|
261
|
+
`DJANGO_QSTASH_DOMAIN`: Must be a valid and publicly accessible domain. For example `https://djangoqstash.com`
|
|
262
|
+
|
|
263
|
+
In development mode, we recommend using a tunnel like [Cloudflare Tunnels](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) with a domain name you control. You can also consider [ngrok](https://ngrok.com/).
|
|
264
|
+
|
|
265
|
+
`DJANGO_QSTASH_WEBHOOK_PATH` (default:`/qstash/webhook/`): The path where QStash will send webhooks to your Django application.
|
|
266
|
+
|
|
267
|
+
`DJANGO_QSTASH_FORCE_HTTPS` (default:`True`): Whether to force HTTPS for the webhook.
|
|
268
|
+
|
|
269
|
+
`DJANGO_QSTASH_RESULT_TTL` (default:`604800`): A number of seconds after which task result data can be safely deleted. Defaults to 604800 seconds (7 days or 7 * 24 * 60 * 60).
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
## Storing Task Results (Optional)
|
|
273
|
+
|
|
274
|
+
In `django_qstash.results.models` we have the `TaskResult` model class that can be used to track async task results. These entries are created via webhooks.
|
|
275
|
+
|
|
276
|
+
To install it, just add `django_qstash.results` to your `INSTALLED_APPS` setting.
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
INSTALLED_APPS = [
|
|
280
|
+
# ...
|
|
281
|
+
"django_qstash.results",
|
|
282
|
+
# ...
|
|
283
|
+
]
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
Run migrations:
|
|
287
|
+
```bash
|
|
288
|
+
python manage.py migrate
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
### Clear Stale Results
|
|
292
|
+
|
|
293
|
+
We recommend purging the `TaskResult` model after a certain amount of time.
|
|
294
|
+
```bash
|
|
295
|
+
python manage.py clear_stale_results
|
|
296
|
+
```
|
|
297
|
+
Args:
|
|
298
|
+
- `--since` is the number of seconds ago to clear results for. Defaults to 604800 seconds (7 days or the `DJANGO_QSTASH_RESULT_TTL` setting).
|
|
299
|
+
- `--no-input` is a flag to skip the confirmation prompt to delete the results.
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
## Definitions
|
|
304
|
+
|
|
305
|
+
- **Background Task**: A function or task that is not part of the request/response cycle.
|
|
306
|
+
- Examples include as sending an email, running a report, or updating a database.
|
|
307
|
+
- Pro: Background tasks can drastically improve the end-user experience since they can move on with their day while the task runs in the background.
|
|
308
|
+
- Con: Processes that run background tasks (like Celery) typically have to run 24/7.
|
|
309
|
+
- **Scale-to-Zero**: Depending on the amount of traffic, Django can be effectively turned off. If done right, when more traffic comes in, Django can be turned back on very quickly.
|
|
310
|
+
- **Serverless**: A cloud computing model where code runs without server management, with scaling and billing tied to usage. Often used interchangeably with "scale-to-zero".
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
## Motivation
|
|
314
|
+
|
|
315
|
+
TLDR - Celery cannot be serverless. I want serverless "Celery" so I only pay for the apps that have attention and traffic. Upstash created QStash to help solve the problem of message queues in a serverless environment. django-qstash is the goldilocks that combines the functionality of Celery with the functionality of QStash all to unlock fully serverless Django.
|
|
316
|
+
|
|
317
|
+
I run a lot of side projects with Django. Some as demos for tutorials based on my work at [@codingforentrepreneurs](https://cfe.sh/github) and some are new businesses that haven't found much traction yet.
|
|
318
|
+
|
|
319
|
+
Most web apps can benefit from async background tasks such as sending emails, running reports, or updating databases.
|
|
320
|
+
|
|
321
|
+
But how?
|
|
322
|
+
|
|
323
|
+
Traditionally, I'd reach for Celery but that can get expensive really quick. Running a lot of Django projects can add up too -- "death by a thousand cuts" if you will. A server for Django, for celery worker, for celery beat scheduler, and so on. It adds up fast.
|
|
324
|
+
|
|
325
|
+
I think serverless is the answer. Pay for what you use and scale to zero when you don't need it and scale up when you do -- all automated.
|
|
326
|
+
|
|
327
|
+
Django can be serverless and is pretty easy to do thanks to Docker and the countless hosting options and services out there. Celery cannot be serverless, at least yet.
|
|
328
|
+
|
|
329
|
+
Let's face it. Celery is a powerful tool to run async background tasks but it comes at a cost. It needs at least one server running 24/7. For best performance it needs 2 (one worker, one beat). It also needs Redis or RabbitMQ. Most background processes that are tied to web apps are not serverless; they have to "listen" for their next task.
|
|
330
|
+
|
|
331
|
+
To make Django truly scale-to-zero and serverless, we need to drop Celery.
|
|
332
|
+
|
|
333
|
+
Enter __django-qstash__.
|
|
334
|
+
|
|
335
|
+
django-qstash is designed to be a near drop-in replacement for Celery's `shared_task` decorator.
|
|
336
|
+
|
|
337
|
+
It works by leveraging Upstash QStash to deliver messages about your tasks (e.g. the function's arguments) via webhooks to your Django application. In the QStash [docs](https://upstash.com/docs/qstash/overall/getstarted), it is described as:
|
|
338
|
+
|
|
339
|
+
> QStash is a serverless messaging and scheduling solution. It fits easily into your existing workflow and allows you to build reliable systems without managing infrastructure.
|
|
340
|
+
>
|
|
341
|
+
> Instead of calling an endpoint directly, QStash acts as a middleman between you and an API to guarantee delivery, perform automatic retries on failure, and more.
|
|
342
|
+
|
|
343
|
+
django-qstash has a webhook handler that converts a QStash message to run a specific `@shared_task` function (the one that called `.delay()` or `.apply_async()`). It's easy, it's cheap, it's effective, and best of all, it unlocks the scale-to-zero potential of Django as a serverless app.
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
> :warning: **BETA Software**: Working on being production-ready soon.
|
|
2
|
+
|
|
3
|
+
# django-qstash
|
|
4
|
+
|
|
5
|
+
_django-qstash_ is a drop-in replacement for Celery's `shared_task`.
|
|
6
|
+
|
|
7
|
+
To do this, we use:
|
|
8
|
+
|
|
9
|
+
- [Upstash QStash](https://upstash.com/docs/qstash/overall/getstarted)
|
|
10
|
+
- A single public _webhook_ to call `@shared_task` functions automatically
|
|
11
|
+
|
|
12
|
+
This allows us to:
|
|
13
|
+
|
|
14
|
+
- Focus just on Django
|
|
15
|
+
- Drop Celery
|
|
16
|
+
- Truly scale Django to zero
|
|
17
|
+
- Run background tasks through webhooks
|
|
18
|
+
- Cut costs
|
|
19
|
+
- Trigger GitHub Actions Workflows or GitLab CI/CD pipelines for handling other kinds of background tasks based on our project's code.
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## Table of Contents
|
|
23
|
+
|
|
24
|
+
- [django-qstash](#django-qstash)
|
|
25
|
+
- [Table of Contents](#table-of-contents)
|
|
26
|
+
- [Installation](#installation)
|
|
27
|
+
- [Using Pip](#using-pip)
|
|
28
|
+
- [Update Settings (`settings.py`)](#update-settings-settingspy)
|
|
29
|
+
- [Configure Webhook URL](#configure-webhook-url)
|
|
30
|
+
- [Required Environment Variables](#required-environment-variables)
|
|
31
|
+
- [Sample Project](#sample-project)
|
|
32
|
+
- [Dependencies](#dependencies)
|
|
33
|
+
- [Usage](#usage)
|
|
34
|
+
- [Define a Task](#define-a-task)
|
|
35
|
+
- [Regular Task Call](#regular-task-call)
|
|
36
|
+
- [Async Task](#async-task)
|
|
37
|
+
- [`.delay()`](#delay)
|
|
38
|
+
- [`.apply_async()`](#apply_async)
|
|
39
|
+
- [`.apply_async()` With Time Delay](#apply_async-with-time-delay)
|
|
40
|
+
- [JSON-ready Arguments](#json-ready-arguments)
|
|
41
|
+
- [Example Task](#example-task)
|
|
42
|
+
- [Configuration](#configuration)
|
|
43
|
+
- [Storing Task Results (Optional)](#storing-task-results-optional)
|
|
44
|
+
- [Clear Stale Results](#clear-stale-results)
|
|
45
|
+
- [Definitions](#definitions)
|
|
46
|
+
- [Motivation](#motivation)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
## Installation
|
|
50
|
+
|
|
51
|
+
### Using Pip
|
|
52
|
+
```bash
|
|
53
|
+
pip install django-qstash
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Update Settings (`settings.py`)
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
INSTALLED_APPS = [
|
|
60
|
+
##...
|
|
61
|
+
"django_qstash",
|
|
62
|
+
"django_qstash.results",
|
|
63
|
+
##...
|
|
64
|
+
]
|
|
65
|
+
```
|
|
66
|
+
- `django_qstash` Includes the `@shared_task` decorator and webhook view
|
|
67
|
+
- `django_qstash.results` (Optional): Store task results in Django DB
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
### Configure Webhook URL
|
|
71
|
+
|
|
72
|
+
In your `ROOT_URLCONF` (e.g. `urls.py`), add the following:
|
|
73
|
+
```python
|
|
74
|
+
from django_qstash.views import qstash_webhook_view
|
|
75
|
+
|
|
76
|
+
urlpatterns = [
|
|
77
|
+
# ...
|
|
78
|
+
path("qstash/webhook/", qstash_webhook_view),
|
|
79
|
+
# ...
|
|
80
|
+
]
|
|
81
|
+
```
|
|
82
|
+
Be sure to use this path in your `DJANGO_QSTASH_WEBHOOK_PATH` environment variable.
|
|
83
|
+
|
|
84
|
+
### Required Environment Variables
|
|
85
|
+
|
|
86
|
+
Get your QStash token and signing keys from [Upstash](https://upstash.com/).
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
QSTASH_TOKEN = "your_token"
|
|
90
|
+
QSTASH_CURRENT_SIGNING_KEY = "your_current_signing_key"
|
|
91
|
+
QSTASH_NEXT_SIGNING_KEY = "your_next_signing_key"
|
|
92
|
+
|
|
93
|
+
# required for django-qstash
|
|
94
|
+
DJANGO_QSTASH_DOMAIN = "https://example.com"
|
|
95
|
+
DJANGO_QSTASH_WEBHOOK_PATH = "/qstash/webhook/"
|
|
96
|
+
```
|
|
97
|
+
> Review [.env.sample](.env.sample) to see all the environment variables you need to set.
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
## Sample Project
|
|
101
|
+
There is a sample project in [sample_project/](sample_project/) that shows how all this is implemented.
|
|
102
|
+
|
|
103
|
+
## Dependencies
|
|
104
|
+
|
|
105
|
+
- [Python 3.10+](https://www.python.org/)
|
|
106
|
+
- [Django 5+](https://docs.djangoproject.com/)
|
|
107
|
+
- [qstash-py](https://github.com/upstash/qstash-py)
|
|
108
|
+
- [Upstash](https://upstash.com/) account
|
|
109
|
+
|
|
110
|
+
## Usage
|
|
111
|
+
|
|
112
|
+
Django-QStash revolves around the `shared_task` decorator. The goal is to be a drop-in replacement for Celery's `shared_task` decorator.
|
|
113
|
+
|
|
114
|
+
Here's how it works:
|
|
115
|
+
- Define a Task
|
|
116
|
+
- Call a Task with `.delay()` or `.apply_async()`
|
|
117
|
+
|
|
118
|
+
### Define a Task
|
|
119
|
+
```python
|
|
120
|
+
from django_qstash import shared_task
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@shared_task
|
|
124
|
+
def hello_world(name: str, age: int = None, activity: str = None):
|
|
125
|
+
if age is None:
|
|
126
|
+
print(f"Hello {name}! I see you're {activity}.")
|
|
127
|
+
return
|
|
128
|
+
print(f"Hello {name}! I see you're {activity} at {age} years old.")
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
### Regular Task Call
|
|
133
|
+
Nothing special here. Just call the function like any other to verify it works.
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
# normal function call
|
|
137
|
+
hello_world("Tony Stark", age=40, activity="building in a cave with a box of scraps.")
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Async Task
|
|
141
|
+
|
|
142
|
+
Using `.delay()` or `.apply_async()` is how you call an async task. This is modeled after Celery and it works as you'd expect.
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
#### `.delay()`
|
|
146
|
+
```python
|
|
147
|
+
hello_world.delay(
|
|
148
|
+
"Tony Stark", age=40, activity="building in a cave with a box of scraps."
|
|
149
|
+
)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
#### `.apply_async()`
|
|
153
|
+
```python
|
|
154
|
+
hello_world.apply_async(
|
|
155
|
+
args=("Tony Stark",),
|
|
156
|
+
kwargs={"activity": "building in a cave with a box of scraps."},
|
|
157
|
+
)
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
#### `.apply_async()` With Time Delay
|
|
161
|
+
|
|
162
|
+
Just use the `countdown` parameter to delay the task by N seconds. (always in seconds): `.apply_async(*args, **kwargs, countdown=N)`
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
# async task delayed 35 seconds
|
|
167
|
+
delay_35_seconds = 35
|
|
168
|
+
hello_world.apply_async(
|
|
169
|
+
args=("Tony Stark",),
|
|
170
|
+
kwargs={"activity": "building in a cave with a box of scraps."},
|
|
171
|
+
countdown=delay_35_seconds,
|
|
172
|
+
)
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### JSON-ready Arguments
|
|
176
|
+
|
|
177
|
+
Each argument needs to be _JSON_ serializable. The way you find out:
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
import json
|
|
181
|
+
|
|
182
|
+
data = {
|
|
183
|
+
"args": ("Tony Stark",),
|
|
184
|
+
"kwargs": {"activity": "building in a cave with a box of scraps."},
|
|
185
|
+
}
|
|
186
|
+
print(json.dumps(data))
|
|
187
|
+
# no errors, you're good to go.
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Example Task
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
# from celery import shared_task
|
|
194
|
+
from django_qstash import shared_task
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@shared_task
|
|
198
|
+
def math_add_task(a, b, save_to_file=False, *args, **kwargs):
|
|
199
|
+
logger.info(f"Adding {a} and {b}")
|
|
200
|
+
if save_to_file:
|
|
201
|
+
with open("math-add-result.txt", "w") as f:
|
|
202
|
+
f.write(f"{a} + {b} = {a + b}")
|
|
203
|
+
return a + b
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
Calling:
|
|
208
|
+
```python
|
|
209
|
+
math_add_task.apply_async(args=(12, 454), save_to_file=True)
|
|
210
|
+
```
|
|
211
|
+
is the same as
|
|
212
|
+
```python
|
|
213
|
+
math_add_task.delay(12, 454, save_to_file=True)
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
But if you need to delay the task, use `.apply_async()` with the `countdown` parameter.
|
|
217
|
+
|
|
218
|
+
```python
|
|
219
|
+
five_hours = 5 * 60 * 60
|
|
220
|
+
math_add_task.apply_async(
|
|
221
|
+
args=(12, 454), kwargs={"save_to_file": True}, countdown=five_hours
|
|
222
|
+
)
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
The `.delay()` method does not support a countdown parameter because it simply passes the arguments (*args, **kwargs) to the `apply_async()` method.
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
## Configuration
|
|
229
|
+
|
|
230
|
+
In Django settings, you can configure the following:
|
|
231
|
+
|
|
232
|
+
`DJANGO_QSTASH_DOMAIN`: Must be a valid and publicly accessible domain. For example `https://djangoqstash.com`
|
|
233
|
+
|
|
234
|
+
In development mode, we recommend using a tunnel like [Cloudflare Tunnels](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) with a domain name you control. You can also consider [ngrok](https://ngrok.com/).
|
|
235
|
+
|
|
236
|
+
`DJANGO_QSTASH_WEBHOOK_PATH` (default:`/qstash/webhook/`): The path where QStash will send webhooks to your Django application.
|
|
237
|
+
|
|
238
|
+
`DJANGO_QSTASH_FORCE_HTTPS` (default:`True`): Whether to force HTTPS for the webhook.
|
|
239
|
+
|
|
240
|
+
`DJANGO_QSTASH_RESULT_TTL` (default:`604800`): A number of seconds after which task result data can be safely deleted. Defaults to 604800 seconds (7 days or 7 * 24 * 60 * 60).
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
## Storing Task Results (Optional)
|
|
244
|
+
|
|
245
|
+
In `django_qstash.results.models` we have the `TaskResult` model class that can be used to track async task results. These entries are created via webhooks.
|
|
246
|
+
|
|
247
|
+
To install it, just add `django_qstash.results` to your `INSTALLED_APPS` setting.
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
INSTALLED_APPS = [
|
|
251
|
+
# ...
|
|
252
|
+
"django_qstash.results",
|
|
253
|
+
# ...
|
|
254
|
+
]
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Run migrations:
|
|
258
|
+
```bash
|
|
259
|
+
python manage.py migrate
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
### Clear Stale Results
|
|
263
|
+
|
|
264
|
+
We recommend purging the `TaskResult` model after a certain amount of time.
|
|
265
|
+
```bash
|
|
266
|
+
python manage.py clear_stale_results
|
|
267
|
+
```
|
|
268
|
+
Args:
|
|
269
|
+
- `--since` is the number of seconds ago to clear results for. Defaults to 604800 seconds (7 days or the `DJANGO_QSTASH_RESULT_TTL` setting).
|
|
270
|
+
- `--no-input` is a flag to skip the confirmation prompt to delete the results.
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
## Definitions
|
|
275
|
+
|
|
276
|
+
- **Background Task**: A function or task that is not part of the request/response cycle.
|
|
277
|
+
- Examples include as sending an email, running a report, or updating a database.
|
|
278
|
+
- Pro: Background tasks can drastically improve the end-user experience since they can move on with their day while the task runs in the background.
|
|
279
|
+
- Con: Processes that run background tasks (like Celery) typically have to run 24/7.
|
|
280
|
+
- **Scale-to-Zero**: Depending on the amount of traffic, Django can be effectively turned off. If done right, when more traffic comes in, Django can be turned back on very quickly.
|
|
281
|
+
- **Serverless**: A cloud computing model where code runs without server management, with scaling and billing tied to usage. Often used interchangeably with "scale-to-zero".
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
## Motivation
|
|
285
|
+
|
|
286
|
+
TLDR - Celery cannot be serverless. I want serverless "Celery" so I only pay for the apps that have attention and traffic. Upstash created QStash to help solve the problem of message queues in a serverless environment. django-qstash is the goldilocks that combines the functionality of Celery with the functionality of QStash all to unlock fully serverless Django.
|
|
287
|
+
|
|
288
|
+
I run a lot of side projects with Django. Some as demos for tutorials based on my work at [@codingforentrepreneurs](https://cfe.sh/github) and some are new businesses that haven't found much traction yet.
|
|
289
|
+
|
|
290
|
+
Most web apps can benefit from async background tasks such as sending emails, running reports, or updating databases.
|
|
291
|
+
|
|
292
|
+
But how?
|
|
293
|
+
|
|
294
|
+
Traditionally, I'd reach for Celery but that can get expensive really quick. Running a lot of Django projects can add up too -- "death by a thousand cuts" if you will. A server for Django, for celery worker, for celery beat scheduler, and so on. It adds up fast.
|
|
295
|
+
|
|
296
|
+
I think serverless is the answer. Pay for what you use and scale to zero when you don't need it and scale up when you do -- all automated.
|
|
297
|
+
|
|
298
|
+
Django can be serverless and is pretty easy to do thanks to Docker and the countless hosting options and services out there. Celery cannot be serverless, at least yet.
|
|
299
|
+
|
|
300
|
+
Let's face it. Celery is a powerful tool to run async background tasks but it comes at a cost. It needs at least one server running 24/7. For best performance it needs 2 (one worker, one beat). It also needs Redis or RabbitMQ. Most background processes that are tied to web apps are not serverless; they have to "listen" for their next task.
|
|
301
|
+
|
|
302
|
+
To make Django truly scale-to-zero and serverless, we need to drop Celery.
|
|
303
|
+
|
|
304
|
+
Enter __django-qstash__.
|
|
305
|
+
|
|
306
|
+
django-qstash is designed to be a near drop-in replacement for Celery's `shared_task` decorator.
|
|
307
|
+
|
|
308
|
+
It works by leveraging Upstash QStash to deliver messages about your tasks (e.g. the function's arguments) via webhooks to your Django application. In the QStash [docs](https://upstash.com/docs/qstash/overall/getstarted), it is described as:
|
|
309
|
+
|
|
310
|
+
> QStash is a serverless messaging and scheduling solution. It fits easily into your existing workflow and allows you to build reliable systems without managing infrastructure.
|
|
311
|
+
>
|
|
312
|
+
> Instead of calling an endpoint directly, QStash acts as a middleman between you and an API to guarantee delivery, perform automatic retries on failure, and more.
|
|
313
|
+
|
|
314
|
+
django-qstash has a webhook handler that converts a QStash message to run a specific `@shared_task` function (the one that called `.delay()` or `.apply_async()`). It's easy, it's cheap, it's effective, and best of all, it unlocks the scale-to-zero potential of Django as a serverless app.
|
|
@@ -6,7 +6,7 @@ requires = [
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "django-qstash"
|
|
9
|
-
version = "0.0.
|
|
9
|
+
version = "0.0.4"
|
|
10
10
|
description = "A drop-in replacement for Celery's shared_task with Upstash QStash."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
license = { file = "LICENSE" }
|
|
@@ -72,6 +72,9 @@ source = [
|
|
|
72
72
|
"django_qstash",
|
|
73
73
|
"tests",
|
|
74
74
|
]
|
|
75
|
+
omit = [
|
|
76
|
+
"*/migrations/*",
|
|
77
|
+
]
|
|
75
78
|
|
|
76
79
|
[tool.coverage.paths]
|
|
77
80
|
source = [
|