django-tasks-google 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_tasks_google-0.1.0/LICENSE +21 -0
- django_tasks_google-0.1.0/PKG-INFO +11 -0
- django_tasks_google-0.1.0/README.md +120 -0
- django_tasks_google-0.1.0/django_tasks_google/__init__.py +0 -0
- django_tasks_google-0.1.0/django_tasks_google/admin.py +181 -0
- django_tasks_google-0.1.0/django_tasks_google/backends.py +150 -0
- django_tasks_google-0.1.0/django_tasks_google/executor.py +60 -0
- django_tasks_google-0.1.0/django_tasks_google/management/__init__.py +0 -0
- django_tasks_google-0.1.0/django_tasks_google/management/commands/__init__.py +0 -0
- django_tasks_google-0.1.0/django_tasks_google/management/commands/execute_task.py +14 -0
- django_tasks_google-0.1.0/django_tasks_google/management/commands/sync_scheduled_tasks.py +8 -0
- django_tasks_google-0.1.0/django_tasks_google/migrations/0001_initial.py +92 -0
- django_tasks_google-0.1.0/django_tasks_google/migrations/__init__.py +0 -0
- django_tasks_google-0.1.0/django_tasks_google/models.py +133 -0
- django_tasks_google-0.1.0/django_tasks_google/scheduler.py +103 -0
- django_tasks_google-0.1.0/django_tasks_google/tasks.py +6 -0
- django_tasks_google-0.1.0/django_tasks_google/urls.py +9 -0
- django_tasks_google-0.1.0/django_tasks_google/views.py +106 -0
- django_tasks_google-0.1.0/django_tasks_google.egg-info/PKG-INFO +11 -0
- django_tasks_google-0.1.0/django_tasks_google.egg-info/SOURCES.txt +23 -0
- django_tasks_google-0.1.0/django_tasks_google.egg-info/dependency_links.txt +1 -0
- django_tasks_google-0.1.0/django_tasks_google.egg-info/requires.txt +4 -0
- django_tasks_google-0.1.0/django_tasks_google.egg-info/top_level.txt +1 -0
- django_tasks_google-0.1.0/pyproject.toml +16 -0
- django_tasks_google-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 @novucs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-tasks-google
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Django's Task Framework backends for: Cloud Tasks, Cloud Scheduler, and Cloud Run Jobs
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: django>=6.0.0
|
|
8
|
+
Requires-Dist: google-cloud-run>=0.1.0
|
|
9
|
+
Requires-Dist: google-cloud-scheduler>=2.0.0
|
|
10
|
+
Requires-Dist: google-cloud-tasks>=2.0.0
|
|
11
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# django-tasks-google
|
|
2
|
+
|
|
3
|
+
**django-tasks-google** provides seamless integration between Django's Task Framework and
|
|
4
|
+
Google Cloud's serverless infrastructure.
|
|
5
|
+
|
|
6
|
+
### Supported Backends
|
|
7
|
+
|
|
8
|
+
* **Cloud Tasks:** For asynchronous task execution with retries and rate limiting.
|
|
9
|
+
* **Cloud Scheduler:** For cron-style scheduled jobs and recurring tasks.
|
|
10
|
+
* **Cloud Run Jobs:** For long-running or resource-intensive background processing.
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
Install the package via `pip`:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install django-tasks-google
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick Start
|
|
21
|
+
|
|
22
|
+
### 1. Register the Application
|
|
23
|
+
|
|
24
|
+
Add `django_tasks_google` to your `INSTALLED_APPS` in `settings.py`:
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
# settings.py
|
|
28
|
+
INSTALLED_APPS = [
|
|
29
|
+
# ...
|
|
30
|
+
"django_tasks_google",
|
|
31
|
+
]
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### 2. Configure Task Backends
|
|
35
|
+
|
|
36
|
+
Define your task execution strategy within the `TASKS` setting. You can mix and match backends based on your
|
|
37
|
+
architectural needs.
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
# settings.py
|
|
41
|
+
TASKS = {
|
|
42
|
+
"default": {
|
|
43
|
+
"BACKEND": "django_tasks_google.backends.CloudTasksBackend",
|
|
44
|
+
"QUEUES": [],
|
|
45
|
+
"OPTIONS": {
|
|
46
|
+
"project_id": "YOUR_PROJECT_ID",
|
|
47
|
+
"location": "us-central1",
|
|
48
|
+
"target_url": "https://your-app.run.app/tasks/execute/",
|
|
49
|
+
"oidc_service_account": "task-invoker@YOUR_PROJECT_ID.iam.gserviceaccount.com",
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
"jobs": {
|
|
53
|
+
"BACKEND": "django_tasks_google.backends.CloudRunJobsBackend",
|
|
54
|
+
"QUEUES": [],
|
|
55
|
+
"OPTIONS": {
|
|
56
|
+
"project_id": "YOUR_PROJECT_ID",
|
|
57
|
+
"location": "us-central1",
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
"scheduler": {
|
|
61
|
+
"BACKEND": "django_tasks_google.backends.CloudSchedulerBackend",
|
|
62
|
+
"QUEUES": [],
|
|
63
|
+
"OPTIONS": {
|
|
64
|
+
"project_id": "YOUR_PROJECT_ID",
|
|
65
|
+
"location": "us-central1",
|
|
66
|
+
"target_url": "https://your-app.run.app/tasks/execute/",
|
|
67
|
+
"oidc_service_account": "task-invoker@YOUR_PROJECT_ID.iam.gserviceaccount.com",
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
}
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
No IAM roles are needed for the `oidc_service_account`. The backend is designed
|
|
74
|
+
to work with public Cloud Run services by performing manual OIDC token
|
|
75
|
+
verification. This ensures that even though the endpoint is publicly accessible,
|
|
76
|
+
only requests signed by your specific service account can trigger task execution.
|
|
77
|
+
|
|
78
|
+
### 3. Register the task execution URLs
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
# urls.py
|
|
82
|
+
from django.urls import path, include
|
|
83
|
+
|
|
84
|
+
urlpatterns = [
|
|
85
|
+
# ...
|
|
86
|
+
path('tasks/', include('django_tasks_google.urls')),
|
|
87
|
+
]
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### 4. Define tasks
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
# tasks.py
|
|
94
|
+
from django.tasks import task
|
|
95
|
+
from django_tasks_google.scheduler import schedule_task
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@task(queue_name="your-cloud-task-queue")
|
|
99
|
+
def send_notification(user_id):
|
|
100
|
+
user = User.objects.get(id=user_id)
|
|
101
|
+
# ...
|
|
102
|
+
return f"Notification sent to {user.email}"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@task(backend="jobs", queue_name="your-cloud-run-job")
|
|
106
|
+
def compute_meaning_of_life():
|
|
107
|
+
# ... long running process ...
|
|
108
|
+
return 42
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@task(backend="scheduler") # "queue_name" will be populated with the scheduled task job name
|
|
112
|
+
def send_daily_newsletter(email):
|
|
113
|
+
user = User.objects.get(email=email)
|
|
114
|
+
# ...
|
|
115
|
+
return f"Newsletter sent to {user.email}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# Creates a job on Google Cloud Scheduler.
|
|
119
|
+
schedule_task(send_daily_newsletter, "0 */3 * * *", args=["user@example.com"])
|
|
120
|
+
```
|
|
File without changes
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import inspect
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from django import forms
|
|
6
|
+
from django.apps import apps
|
|
7
|
+
from django.contrib import admin, messages
|
|
8
|
+
from django.core.exceptions import ValidationError
|
|
9
|
+
|
|
10
|
+
from django_tasks_google.models import ScheduledTask
|
|
11
|
+
from django_tasks_google.scheduler import delete_remote_scheduled_task
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_task_choices():
|
|
15
|
+
choices = []
|
|
16
|
+
for app_config in apps.get_app_configs():
|
|
17
|
+
try:
|
|
18
|
+
module = importlib.import_module(f"{app_config.name}.tasks")
|
|
19
|
+
for name, obj in inspect.getmembers(module):
|
|
20
|
+
is_task_decorated = (
|
|
21
|
+
hasattr(obj, "task")
|
|
22
|
+
or hasattr(obj, "_is_task")
|
|
23
|
+
or type(obj).__name__ == "Task"
|
|
24
|
+
)
|
|
25
|
+
if is_task_decorated:
|
|
26
|
+
path = f"{app_config.name}.tasks.{name}"
|
|
27
|
+
choices.append((path, path))
|
|
28
|
+
except ImportError:
|
|
29
|
+
continue
|
|
30
|
+
return [("", "---------")] + sorted(choices)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ScheduledTaskAdminForm(forms.ModelForm):
|
|
34
|
+
name = forms.CharField(
|
|
35
|
+
help_text="Name can only contain alphanumeric characters, hyphens '-' and underscores '_'"
|
|
36
|
+
)
|
|
37
|
+
task_selector = forms.ChoiceField(
|
|
38
|
+
choices=[],
|
|
39
|
+
required=False,
|
|
40
|
+
label="Select Task (Optional)",
|
|
41
|
+
help_text="Pick a task here OR type a custom path below.",
|
|
42
|
+
)
|
|
43
|
+
schedule = forms.CharField(
|
|
44
|
+
help_text=(
|
|
45
|
+
"Schedules are specified using unix-cron format. "
|
|
46
|
+
'E.g. every minute: "* * * * *", every 3 hours: "0 */3 * * *", every Monday at 9:00: "0 9 * * 1".'
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def __init__(self, *args, **kwargs):
|
|
51
|
+
super().__init__(*args, **kwargs)
|
|
52
|
+
self.fields["task_selector"].choices = [
|
|
53
|
+
("", "--- Manual Entry ---")
|
|
54
|
+
] + get_task_choices()
|
|
55
|
+
self.fields["module_path"].required = False
|
|
56
|
+
self.fields["cloud_scheduler_job_name"].required = False
|
|
57
|
+
|
|
58
|
+
class Meta:
|
|
59
|
+
model = ScheduledTask
|
|
60
|
+
fields = "__all__"
|
|
61
|
+
widgets = {
|
|
62
|
+
"name": forms.TextInput(
|
|
63
|
+
attrs={"style": "width: 400px;", "placeholder": "task-name-here"}
|
|
64
|
+
),
|
|
65
|
+
"module_path": forms.TextInput(attrs={"style": "width: 400px;"}),
|
|
66
|
+
"backend": forms.TextInput(attrs={"style": "width: 200px;"}),
|
|
67
|
+
"schedule": forms.TextInput(attrs={"placeholder": "*/5 * * * *"}),
|
|
68
|
+
"time_zone": forms.TextInput(attrs={"placeholder": "UTC"}),
|
|
69
|
+
"cloud_scheduler_job_name": forms.TextInput(
|
|
70
|
+
attrs={"style": "width: 400px;"}
|
|
71
|
+
),
|
|
72
|
+
"description": forms.Textarea(attrs={"rows": 3, "cols": 40}),
|
|
73
|
+
"args": forms.Textarea(attrs={"rows": 3, "cols": 40}),
|
|
74
|
+
"kwargs": forms.Textarea(attrs={"rows": 3, "cols": 40}),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
def clean(self):
|
|
78
|
+
cleaned_data = super().clean()
|
|
79
|
+
selector_val = cleaned_data.get("task_selector")
|
|
80
|
+
if selector_val:
|
|
81
|
+
cleaned_data["module_path"] = selector_val
|
|
82
|
+
if not cleaned_data["module_path"]:
|
|
83
|
+
raise ValidationError("Either task selector or module path must be set")
|
|
84
|
+
return cleaned_data
|
|
85
|
+
|
|
86
|
+
def clean_name(self):
|
|
87
|
+
name = self.cleaned_data.get("name")
|
|
88
|
+
if not re.match(r"^[a-zA-Z0-9_-]+$", name):
|
|
89
|
+
raise ValidationError(
|
|
90
|
+
"Name can only contain alphanumeric characters, hyphens '-', and underscores '_'."
|
|
91
|
+
)
|
|
92
|
+
return name
|
|
93
|
+
|
|
94
|
+
def clean_args(self):
|
|
95
|
+
data = self.cleaned_data.get("args")
|
|
96
|
+
if data in [None, ""]:
|
|
97
|
+
return []
|
|
98
|
+
if not isinstance(data, list):
|
|
99
|
+
raise ValidationError(
|
|
100
|
+
"Arguments must be a valid JSON list (e.g., [1, 'test'])."
|
|
101
|
+
)
|
|
102
|
+
return data
|
|
103
|
+
|
|
104
|
+
def clean_kwargs(self):
|
|
105
|
+
data = self.cleaned_data.get("kwargs")
|
|
106
|
+
if data in [None, ""]:
|
|
107
|
+
return {}
|
|
108
|
+
if not isinstance(data, dict):
|
|
109
|
+
raise ValidationError(
|
|
110
|
+
'Keyword arguments must be a valid JSON object (e.g., {"key": "value"}).'
|
|
111
|
+
)
|
|
112
|
+
return data
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@admin.register(ScheduledTask)
|
|
116
|
+
class ScheduledTaskAdmin(admin.ModelAdmin):
|
|
117
|
+
form = ScheduledTaskAdminForm
|
|
118
|
+
|
|
119
|
+
list_display = ("name", "state", "schedule", "time_zone", "backend")
|
|
120
|
+
list_filter = ("state", "backend", "time_zone")
|
|
121
|
+
search_fields = ("name", "module_path", "description")
|
|
122
|
+
fieldsets = (
|
|
123
|
+
("General Info", {"fields": ("name", "description", "state")}),
|
|
124
|
+
(
|
|
125
|
+
"Execution Details",
|
|
126
|
+
{"fields": ("task_selector", "module_path", "backend", "takes_context")},
|
|
127
|
+
),
|
|
128
|
+
(
|
|
129
|
+
"Parameters",
|
|
130
|
+
{
|
|
131
|
+
"fields": ("args", "kwargs"),
|
|
132
|
+
"description": "JSON formatted arguments for the task.",
|
|
133
|
+
},
|
|
134
|
+
),
|
|
135
|
+
(
|
|
136
|
+
"Scheduling",
|
|
137
|
+
{"fields": ("schedule", "time_zone", "cloud_scheduler_job_name")},
|
|
138
|
+
),
|
|
139
|
+
)
|
|
140
|
+
actions = ["sync_tasks"]
|
|
141
|
+
|
|
142
|
+
@admin.action(description="Sync selected tasks with Cloud Scheduler")
|
|
143
|
+
def sync_tasks(self, request, queryset):
|
|
144
|
+
for task in queryset:
|
|
145
|
+
try:
|
|
146
|
+
task.sync()
|
|
147
|
+
self.message_user(
|
|
148
|
+
request, f"Successfully synced '{task.name}'", messages.SUCCESS
|
|
149
|
+
)
|
|
150
|
+
except Exception as e:
|
|
151
|
+
self.message_user(
|
|
152
|
+
request, f"Failed to sync '{task.name}': {str(e)}", messages.ERROR
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def save_model(self, request, obj, form, change):
|
|
156
|
+
super().save_model(request, obj, form, change)
|
|
157
|
+
try:
|
|
158
|
+
obj.sync()
|
|
159
|
+
except Exception as e:
|
|
160
|
+
self.message_user(
|
|
161
|
+
request, f"Model saved but sync failed: {e}", messages.WARNING
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
def delete_model(self, request, obj):
|
|
165
|
+
self._cleanup_cloud_scheduler(request, obj)
|
|
166
|
+
super().delete_model(request, obj)
|
|
167
|
+
|
|
168
|
+
def delete_queryset(self, request, queryset):
|
|
169
|
+
for obj in queryset:
|
|
170
|
+
self._cleanup_cloud_scheduler(request, obj)
|
|
171
|
+
super().delete_queryset(request, queryset)
|
|
172
|
+
|
|
173
|
+
def _cleanup_cloud_scheduler(self, request, task):
|
|
174
|
+
try:
|
|
175
|
+
delete_remote_scheduled_task(task)
|
|
176
|
+
except Exception as e:
|
|
177
|
+
self.message_user(
|
|
178
|
+
request,
|
|
179
|
+
f"Cloud Scheduler deletion failed for {task.name}: {e}",
|
|
180
|
+
messages.WARNING,
|
|
181
|
+
)
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
4
|
+
from django.tasks.backends.base import BaseTaskBackend
|
|
5
|
+
from django.tasks.signals import task_enqueued
|
|
6
|
+
|
|
7
|
+
from django_tasks_google.models import TaskExecution
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CloudRunJobsBackend(BaseTaskBackend):
|
|
11
|
+
supports_defer = False
|
|
12
|
+
supports_async_task = True
|
|
13
|
+
supports_get_result = True
|
|
14
|
+
supports_priority = False
|
|
15
|
+
|
|
16
|
+
def __init__(self, alias, params):
|
|
17
|
+
super().__init__(alias, params)
|
|
18
|
+
self.project_id = self.options.get("project_id")
|
|
19
|
+
self.location = self.options.get("location")
|
|
20
|
+
if not self.project_id:
|
|
21
|
+
raise ImproperlyConfigured("project_id is required")
|
|
22
|
+
if not self.location:
|
|
23
|
+
raise ImproperlyConfigured("location is required")
|
|
24
|
+
|
|
25
|
+
def enqueue(self, task, args, kwargs):
|
|
26
|
+
from google.cloud import run_v2
|
|
27
|
+
|
|
28
|
+
self.validate_task(task)
|
|
29
|
+
execution = TaskExecution.objects.create(
|
|
30
|
+
priority=task.priority,
|
|
31
|
+
module_path=task.module_path,
|
|
32
|
+
backend=self.alias,
|
|
33
|
+
queue_name=task.queue_name,
|
|
34
|
+
run_after=task.run_after,
|
|
35
|
+
takes_context=task.takes_context,
|
|
36
|
+
args=list(args),
|
|
37
|
+
kwargs=dict(kwargs),
|
|
38
|
+
)
|
|
39
|
+
client = run_v2.JobsClient()
|
|
40
|
+
request = run_v2.RunJobRequest(
|
|
41
|
+
name=f"projects/{self.project_id}/locations/{self.location}/jobs/{task.queue_name}", # type: ignore
|
|
42
|
+
overrides=run_v2.RunJobRequest.Overrides( # type: ignore
|
|
43
|
+
container_overrides=[ # type: ignore
|
|
44
|
+
run_v2.RunJobRequest.Overrides.ContainerOverride(
|
|
45
|
+
args=["python", "manage.py", "execute_task", str(execution.pk)] # type: ignore
|
|
46
|
+
)
|
|
47
|
+
]
|
|
48
|
+
),
|
|
49
|
+
)
|
|
50
|
+
operation = client.run_job(request=request)
|
|
51
|
+
execution.cloud_run_job_execution_name = operation.metadata.name
|
|
52
|
+
execution.save(update_fields=["cloud_run_job_execution_name"])
|
|
53
|
+
task_result = execution.task_result
|
|
54
|
+
task_enqueued.send(sender=type(self), task_result=task_result)
|
|
55
|
+
return task_result
|
|
56
|
+
|
|
57
|
+
def get_result(self, result_id):
|
|
58
|
+
return TaskExecution.objects.get(pk=result_id).task_result
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class CloudTasksBackend(BaseTaskBackend):
|
|
62
|
+
supports_defer = True
|
|
63
|
+
supports_async_task = True
|
|
64
|
+
supports_get_result = True
|
|
65
|
+
supports_priority = False
|
|
66
|
+
|
|
67
|
+
def __init__(self, alias, params):
|
|
68
|
+
super().__init__(alias, params)
|
|
69
|
+
self.project_id = self.options.get("project_id")
|
|
70
|
+
self.location = self.options.get("location")
|
|
71
|
+
self.target_url = self.options.get("target_url")
|
|
72
|
+
self.oidc_service_account = self.options.get("oidc_service_account")
|
|
73
|
+
if not self.project_id:
|
|
74
|
+
raise ImproperlyConfigured("project_id is required")
|
|
75
|
+
if not self.location:
|
|
76
|
+
raise ImproperlyConfigured("location is required")
|
|
77
|
+
if not self.target_url:
|
|
78
|
+
raise ImproperlyConfigured("target_url is required")
|
|
79
|
+
if not self.oidc_service_account:
|
|
80
|
+
raise ImproperlyConfigured("oidc_service_account is required")
|
|
81
|
+
|
|
82
|
+
def enqueue(self, task, args, kwargs):
|
|
83
|
+
from google.cloud import tasks_v2
|
|
84
|
+
from google.protobuf import timestamp_pb2
|
|
85
|
+
|
|
86
|
+
self.validate_task(task)
|
|
87
|
+
execution = TaskExecution.objects.create(
|
|
88
|
+
priority=task.priority,
|
|
89
|
+
module_path=task.module_path,
|
|
90
|
+
backend=self.alias,
|
|
91
|
+
queue_name=task.queue_name,
|
|
92
|
+
run_after=task.run_after,
|
|
93
|
+
takes_context=task.takes_context,
|
|
94
|
+
args=list(args),
|
|
95
|
+
kwargs=dict(kwargs),
|
|
96
|
+
)
|
|
97
|
+
client = tasks_v2.CloudTasksClient()
|
|
98
|
+
payload = {"backend": self.alias, "task_execution_id": execution.pk}
|
|
99
|
+
cloud_task_definition = tasks_v2.Task(
|
|
100
|
+
http_request=tasks_v2.HttpRequest( # type: ignore
|
|
101
|
+
http_method=tasks_v2.HttpMethod.POST, # type: ignore
|
|
102
|
+
url=self.target_url,
|
|
103
|
+
headers={"Content-Type": "application/json"},
|
|
104
|
+
body=json.dumps(payload).encode(), # type: ignore
|
|
105
|
+
oidc_token=tasks_v2.OidcToken( # type: ignore
|
|
106
|
+
service_account_email=self.oidc_service_account,
|
|
107
|
+
audience=self.target_url,
|
|
108
|
+
),
|
|
109
|
+
),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if task.run_after:
|
|
113
|
+
schedule_time = timestamp_pb2.Timestamp()
|
|
114
|
+
schedule_time.FromDatetime(task.run_after)
|
|
115
|
+
cloud_task_definition["schedule_time"] = schedule_time
|
|
116
|
+
|
|
117
|
+
cloud_task = client.create_task(
|
|
118
|
+
parent=f"projects/{self.project_id}/locations/{self.location}/queues/{task.queue_name}",
|
|
119
|
+
task=cloud_task_definition,
|
|
120
|
+
)
|
|
121
|
+
execution.cloud_task_name = cloud_task.name
|
|
122
|
+
execution.save(update_fields=["cloud_task_name"])
|
|
123
|
+
task_result = execution.task_result
|
|
124
|
+
task_enqueued.send(sender=type(self), task_result=task_result)
|
|
125
|
+
return task_result
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class CloudSchedulerBackend(BaseTaskBackend):
|
|
129
|
+
supports_defer = False
|
|
130
|
+
supports_async_task = True
|
|
131
|
+
supports_get_result = False
|
|
132
|
+
supports_priority = False
|
|
133
|
+
|
|
134
|
+
def __init__(self, alias, params):
|
|
135
|
+
super().__init__(alias, params)
|
|
136
|
+
self.project_id = self.options.get("project_id")
|
|
137
|
+
self.location = self.options.get("location")
|
|
138
|
+
self.target_url = self.options.get("target_url")
|
|
139
|
+
self.oidc_service_account = self.options.get("oidc_service_account")
|
|
140
|
+
if not self.project_id:
|
|
141
|
+
raise ImproperlyConfigured("project_id is required")
|
|
142
|
+
if not self.location:
|
|
143
|
+
raise ImproperlyConfigured("location is required")
|
|
144
|
+
if not self.target_url:
|
|
145
|
+
raise ImproperlyConfigured("target_url is required")
|
|
146
|
+
if not self.oidc_service_account:
|
|
147
|
+
raise ImproperlyConfigured("oidc_service_account is required")
|
|
148
|
+
|
|
149
|
+
def enqueue(self, task, args, kwargs):
|
|
150
|
+
raise NotImplementedError("This task my only be enqueued by Cloud Scheduler")
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from traceback import format_exception
|
|
3
|
+
|
|
4
|
+
from django.db import close_old_connections
|
|
5
|
+
from django.tasks import task_backends
|
|
6
|
+
from django.tasks.base import TaskContext, TaskResultStatus
|
|
7
|
+
from django.tasks.signals import task_finished, task_started
|
|
8
|
+
from django.utils import timezone
|
|
9
|
+
|
|
10
|
+
from django_tasks_google.models import TaskExecution
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger("django_tasks_google")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def execute_task(execution: TaskExecution):
|
|
16
|
+
worker_id = timezone.now().isoformat()
|
|
17
|
+
|
|
18
|
+
logger.info(f"Running {execution}")
|
|
19
|
+
sender = type(task_backends[execution.backend])
|
|
20
|
+
execution.last_attempted_at = timezone.now()
|
|
21
|
+
execution.status = TaskResultStatus.RUNNING
|
|
22
|
+
execution.worker_ids.append(worker_id)
|
|
23
|
+
|
|
24
|
+
if not execution.started_at:
|
|
25
|
+
execution.started_at = execution.last_attempted_at
|
|
26
|
+
execution.save()
|
|
27
|
+
task_started.send(sender=sender, task_result=execution.task_result)
|
|
28
|
+
else:
|
|
29
|
+
execution.save()
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
args = execution.args
|
|
33
|
+
if execution.takes_context:
|
|
34
|
+
args = (TaskContext(task_result=execution.task_result), *args)
|
|
35
|
+
return_value = execution.task.call(*args, **execution.kwargs)
|
|
36
|
+
|
|
37
|
+
close_old_connections()
|
|
38
|
+
execution.finished_at = timezone.now()
|
|
39
|
+
execution.status = TaskResultStatus.SUCCESSFUL
|
|
40
|
+
execution.return_value = return_value
|
|
41
|
+
execution.save()
|
|
42
|
+
logger.info(f"Completed {execution}")
|
|
43
|
+
task_finished.send(sender=sender, task_result=execution.task_result)
|
|
44
|
+
return True
|
|
45
|
+
|
|
46
|
+
except Exception as e:
|
|
47
|
+
close_old_connections()
|
|
48
|
+
execution.finished_at = timezone.now()
|
|
49
|
+
execution.status = TaskResultStatus.FAILED
|
|
50
|
+
exception_type = type(e)
|
|
51
|
+
execution.errors.append(
|
|
52
|
+
{
|
|
53
|
+
"exception_class_path": f"{exception_type.__module__}.{exception_type.__qualname__}",
|
|
54
|
+
"traceback": "".join(format_exception(e)),
|
|
55
|
+
}
|
|
56
|
+
)
|
|
57
|
+
execution.save()
|
|
58
|
+
logger.exception(f"An error occurred during {execution}")
|
|
59
|
+
task_finished.send(sender=sender, task_result=execution.task_result)
|
|
60
|
+
return False
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
2
|
+
|
|
3
|
+
from django_tasks_google.executor import execute_task
|
|
4
|
+
from django_tasks_google.models import TaskExecution
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Command(BaseCommand):
|
|
8
|
+
def add_arguments(self, parser):
|
|
9
|
+
parser.add_argument("task_execution_id", type=str)
|
|
10
|
+
|
|
11
|
+
def handle(self, *args, **options):
|
|
12
|
+
execution = TaskExecution.objects.get(pk=options["task_execution_id"])
|
|
13
|
+
if not execute_task(execution):
|
|
14
|
+
raise CommandError("Task execution failed")
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from django.db import migrations, models
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Migration(migrations.Migration):
|
|
5
|
+
initial = True
|
|
6
|
+
|
|
7
|
+
dependencies = []
|
|
8
|
+
|
|
9
|
+
operations = [
|
|
10
|
+
migrations.CreateModel(
|
|
11
|
+
name="ScheduledTask",
|
|
12
|
+
fields=[
|
|
13
|
+
(
|
|
14
|
+
"id",
|
|
15
|
+
models.BigAutoField(
|
|
16
|
+
auto_created=True,
|
|
17
|
+
primary_key=True,
|
|
18
|
+
serialize=False,
|
|
19
|
+
verbose_name="ID",
|
|
20
|
+
),
|
|
21
|
+
),
|
|
22
|
+
("name", models.TextField(unique=True)),
|
|
23
|
+
("description", models.TextField(blank=True, default="")),
|
|
24
|
+
("module_path", models.TextField()),
|
|
25
|
+
("backend", models.TextField(default="scheduler")),
|
|
26
|
+
("takes_context", models.BooleanField(default=False)),
|
|
27
|
+
("args", models.JSONField(blank=True, default=list)),
|
|
28
|
+
("kwargs", models.JSONField(blank=True, default=dict)),
|
|
29
|
+
("schedule", models.TextField()),
|
|
30
|
+
("time_zone", models.TextField(default="UTC")),
|
|
31
|
+
("cloud_scheduler_job_name", models.TextField(null=True, unique=True)),
|
|
32
|
+
(
|
|
33
|
+
"state",
|
|
34
|
+
models.TextField(
|
|
35
|
+
choices=[("enabled", "Enabled"), ("disabled", "Disabled")],
|
|
36
|
+
default="enabled",
|
|
37
|
+
),
|
|
38
|
+
),
|
|
39
|
+
],
|
|
40
|
+
),
|
|
41
|
+
migrations.CreateModel(
|
|
42
|
+
name="TaskExecution",
|
|
43
|
+
fields=[
|
|
44
|
+
(
|
|
45
|
+
"id",
|
|
46
|
+
models.BigAutoField(
|
|
47
|
+
auto_created=True,
|
|
48
|
+
primary_key=True,
|
|
49
|
+
serialize=False,
|
|
50
|
+
verbose_name="ID",
|
|
51
|
+
),
|
|
52
|
+
),
|
|
53
|
+
("priority", models.IntegerField(default=0)),
|
|
54
|
+
("module_path", models.TextField()),
|
|
55
|
+
("backend", models.TextField(default="default")),
|
|
56
|
+
("queue_name", models.TextField(default="default")),
|
|
57
|
+
("run_after", models.DateTimeField(null=True)),
|
|
58
|
+
("takes_context", models.BooleanField(default=False)),
|
|
59
|
+
("args", models.JSONField(default=list)),
|
|
60
|
+
("kwargs", models.JSONField(default=dict)),
|
|
61
|
+
(
|
|
62
|
+
"status",
|
|
63
|
+
models.TextField(
|
|
64
|
+
choices=[
|
|
65
|
+
("READY", "Ready"),
|
|
66
|
+
("RUNNING", "Running"),
|
|
67
|
+
("FAILED", "Failed"),
|
|
68
|
+
("SUCCESSFUL", "Successful"),
|
|
69
|
+
],
|
|
70
|
+
default="READY",
|
|
71
|
+
),
|
|
72
|
+
),
|
|
73
|
+
("enqueued_at", models.DateTimeField(auto_now_add=True)),
|
|
74
|
+
("started_at", models.DateTimeField(null=True)),
|
|
75
|
+
("finished_at", models.DateTimeField(null=True)),
|
|
76
|
+
("last_attempted_at", models.DateTimeField(null=True)),
|
|
77
|
+
("errors", models.JSONField(default=list)),
|
|
78
|
+
("worker_ids", models.JSONField(default=list)),
|
|
79
|
+
("return_value", models.JSONField(null=True)),
|
|
80
|
+
("cancelled_at", models.DateTimeField(null=True)),
|
|
81
|
+
(
|
|
82
|
+
"cloud_run_job_execution_name",
|
|
83
|
+
models.TextField(null=True, unique=True),
|
|
84
|
+
),
|
|
85
|
+
("cloud_task_name", models.TextField(null=True, unique=True)),
|
|
86
|
+
(
|
|
87
|
+
"cloud_scheduler_idempotency_key",
|
|
88
|
+
models.TextField(null=True, unique=True),
|
|
89
|
+
),
|
|
90
|
+
],
|
|
91
|
+
),
|
|
92
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
from django.db import models
|
|
2
|
+
from django.tasks import (
|
|
3
|
+
TaskResultStatus,
|
|
4
|
+
Task,
|
|
5
|
+
TaskResult,
|
|
6
|
+
DEFAULT_TASK_QUEUE_NAME,
|
|
7
|
+
task_backends,
|
|
8
|
+
DEFAULT_TASK_BACKEND_ALIAS,
|
|
9
|
+
)
|
|
10
|
+
from django.tasks.base import TaskError, DEFAULT_TASK_PRIORITY
|
|
11
|
+
from django.utils import timezone
|
|
12
|
+
from django.utils.module_loading import import_string
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ScheduledTask(models.Model):
|
|
16
|
+
class State(models.TextChoices):
|
|
17
|
+
ENABLED = "enabled"
|
|
18
|
+
DISABLED = "disabled"
|
|
19
|
+
|
|
20
|
+
name = models.TextField(unique=True)
|
|
21
|
+
description = models.TextField(blank=True, default="")
|
|
22
|
+
module_path = models.TextField()
|
|
23
|
+
backend = models.TextField(default="scheduler")
|
|
24
|
+
takes_context = models.BooleanField(default=False)
|
|
25
|
+
args = models.JSONField(default=list, blank=True)
|
|
26
|
+
kwargs = models.JSONField(default=dict, blank=True)
|
|
27
|
+
|
|
28
|
+
schedule = models.TextField()
|
|
29
|
+
time_zone = models.TextField(default="UTC")
|
|
30
|
+
cloud_scheduler_job_name = models.TextField(null=True, unique=True)
|
|
31
|
+
state = models.TextField(choices=State, default=State.ENABLED)
|
|
32
|
+
|
|
33
|
+
def sync(self):
|
|
34
|
+
from django_tasks_google.scheduler import sync_scheduled_task
|
|
35
|
+
|
|
36
|
+
sync_scheduled_task(self)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class TaskExecution(models.Model):
|
|
40
|
+
priority = models.IntegerField(default=DEFAULT_TASK_PRIORITY)
|
|
41
|
+
module_path = models.TextField()
|
|
42
|
+
backend = models.TextField(default=DEFAULT_TASK_BACKEND_ALIAS)
|
|
43
|
+
queue_name = models.TextField(default=DEFAULT_TASK_QUEUE_NAME)
|
|
44
|
+
run_after = models.DateTimeField(null=True)
|
|
45
|
+
takes_context = models.BooleanField(default=False)
|
|
46
|
+
|
|
47
|
+
args = models.JSONField(default=list)
|
|
48
|
+
kwargs = models.JSONField(default=dict)
|
|
49
|
+
|
|
50
|
+
status = models.TextField(
|
|
51
|
+
choices=TaskResultStatus,
|
|
52
|
+
default=TaskResultStatus.READY.value,
|
|
53
|
+
)
|
|
54
|
+
enqueued_at = models.DateTimeField(auto_now_add=True)
|
|
55
|
+
started_at = models.DateTimeField(null=True)
|
|
56
|
+
finished_at = models.DateTimeField(null=True)
|
|
57
|
+
last_attempted_at = models.DateTimeField(null=True)
|
|
58
|
+
errors = models.JSONField(default=list)
|
|
59
|
+
worker_ids = models.JSONField(default=list)
|
|
60
|
+
|
|
61
|
+
return_value = models.JSONField(null=True)
|
|
62
|
+
|
|
63
|
+
cancelled_at = models.DateTimeField(null=True)
|
|
64
|
+
cloud_run_job_execution_name = models.TextField(null=True, unique=True)
|
|
65
|
+
cloud_task_name = models.TextField(null=True, unique=True)
|
|
66
|
+
cloud_scheduler_idempotency_key = models.TextField(null=True, unique=True)
|
|
67
|
+
|
|
68
|
+
def __str__(self):
|
|
69
|
+
idempotency_key = (
|
|
70
|
+
self.cloud_run_job_execution_name
|
|
71
|
+
or self.cloud_task_name
|
|
72
|
+
or self.cloud_scheduler_idempotency_key
|
|
73
|
+
)
|
|
74
|
+
return f"[{self.status.upper()}] {idempotency_key} (#{self.pk}) at {self.enqueued_at:%Y-%m-%d %H:%M}"
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def task(self) -> Task:
|
|
78
|
+
return Task(
|
|
79
|
+
priority=DEFAULT_TASK_PRIORITY,
|
|
80
|
+
func=import_string(self.module_path).func,
|
|
81
|
+
backend=self.backend,
|
|
82
|
+
queue_name=self.queue_name,
|
|
83
|
+
run_after=self.run_after,
|
|
84
|
+
takes_context=self.takes_context,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def task_result(self) -> TaskResult:
|
|
89
|
+
task_result = TaskResult(
|
|
90
|
+
task=self.task,
|
|
91
|
+
id=str(self.pk),
|
|
92
|
+
status=self.status,
|
|
93
|
+
enqueued_at=self.enqueued_at,
|
|
94
|
+
started_at=self.started_at,
|
|
95
|
+
finished_at=self.finished_at,
|
|
96
|
+
last_attempted_at=self.last_attempted_at,
|
|
97
|
+
args=self.args,
|
|
98
|
+
kwargs=self.kwargs,
|
|
99
|
+
backend=self.backend,
|
|
100
|
+
errors=self.task_errors,
|
|
101
|
+
worker_ids=self.worker_ids,
|
|
102
|
+
)
|
|
103
|
+
object.__setattr__(task_result, "_return_value", self.return_value)
|
|
104
|
+
return task_result
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def task_errors(self) -> list[TaskError]:
|
|
108
|
+
return [
|
|
109
|
+
TaskError(
|
|
110
|
+
exception_class_path=error["exception_class_path"],
|
|
111
|
+
traceback=error["traceback"],
|
|
112
|
+
)
|
|
113
|
+
for error in self.errors
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def backend_class(self):
|
|
118
|
+
return task_backends[self.backend]
|
|
119
|
+
|
|
120
|
+
def cancel(self):
|
|
121
|
+
if not self.cloud_run_job_execution_name:
|
|
122
|
+
raise NotImplementedError("Only Cloud Run Jobs may be cancelled")
|
|
123
|
+
|
|
124
|
+
from google.cloud import run_v2
|
|
125
|
+
|
|
126
|
+
client = run_v2.ExecutionsClient()
|
|
127
|
+
client.cancel_execution(
|
|
128
|
+
run_v2.CancelExecutionRequest(name=self.cloud_run_job_execution_name)
|
|
129
|
+
)
|
|
130
|
+
self.cancelled_at = timezone.now()
|
|
131
|
+
self.finished_at = self.cancelled_at
|
|
132
|
+
self.status = TaskResultStatus.FAILED
|
|
133
|
+
self.save(update_fields=["cancelled_at", "finished_at", "status"])
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from django.db import transaction
|
|
4
|
+
from django.tasks import Task, task_backends
|
|
5
|
+
|
|
6
|
+
from django_tasks_google.models import ScheduledTask
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def schedule_task(
|
|
10
|
+
task: Task,
|
|
11
|
+
schedule: str,
|
|
12
|
+
*,
|
|
13
|
+
name: str = "",
|
|
14
|
+
description: str = "",
|
|
15
|
+
backend: str = "scheduler",
|
|
16
|
+
takes_context: bool = False,
|
|
17
|
+
args: list | None = None,
|
|
18
|
+
kwargs: dict | None = None,
|
|
19
|
+
time_zone: str = "UTC",
|
|
20
|
+
enabled: bool = True,
|
|
21
|
+
) -> ScheduledTask:
|
|
22
|
+
with transaction.atomic():
|
|
23
|
+
scheduled_task = ScheduledTask.objects.create(
|
|
24
|
+
name=name,
|
|
25
|
+
description=description,
|
|
26
|
+
module_path=task.module_path,
|
|
27
|
+
backend=task_backends[backend].alias,
|
|
28
|
+
takes_context=takes_context,
|
|
29
|
+
args=args or [],
|
|
30
|
+
kwargs=kwargs or {},
|
|
31
|
+
schedule=schedule,
|
|
32
|
+
time_zone=time_zone,
|
|
33
|
+
state=ScheduledTask.State.ENABLED
|
|
34
|
+
if enabled
|
|
35
|
+
else ScheduledTask.State.DISABLED,
|
|
36
|
+
)
|
|
37
|
+
scheduled_task.sync()
|
|
38
|
+
return scheduled_task
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def sync_scheduled_tasks():
|
|
42
|
+
for task in ScheduledTask.objects.all():
|
|
43
|
+
sync_scheduled_task(task)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def sync_scheduled_task(task: ScheduledTask):
|
|
47
|
+
from google.api_core.exceptions import NotFound
|
|
48
|
+
from google.cloud import scheduler_v1
|
|
49
|
+
|
|
50
|
+
client = scheduler_v1.CloudSchedulerClient()
|
|
51
|
+
backend = task_backends[task.backend]
|
|
52
|
+
parent = f"projects/{backend.project_id}/locations/{backend.location}"
|
|
53
|
+
job_name = f"{parent}/jobs/{task.name}"
|
|
54
|
+
payload = {"backend": task.backend}
|
|
55
|
+
job = scheduler_v1.Job(
|
|
56
|
+
name=job_name, # type: ignore
|
|
57
|
+
description=task.description,
|
|
58
|
+
schedule=task.schedule,
|
|
59
|
+
time_zone=task.time_zone,
|
|
60
|
+
http_target=scheduler_v1.HttpTarget( # type: ignore
|
|
61
|
+
http_method=scheduler_v1.HttpMethod.POST, # type: ignore
|
|
62
|
+
uri=backend.target_url,
|
|
63
|
+
headers={"Content-Type": "application/json"},
|
|
64
|
+
body=json.dumps(payload).encode(), # type: ignore
|
|
65
|
+
oidc_token=scheduler_v1.OidcToken( # type: ignore
|
|
66
|
+
service_account_email=backend.oidc_service_account,
|
|
67
|
+
audience=backend.target_url,
|
|
68
|
+
),
|
|
69
|
+
),
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
job_exists = False
|
|
73
|
+
try:
|
|
74
|
+
client.get_job(name=job_name)
|
|
75
|
+
job_exists = True
|
|
76
|
+
except NotFound:
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
if job_exists:
|
|
80
|
+
update_mask = {"paths": ["description", "schedule", "time_zone", "http_target"]}
|
|
81
|
+
job = client.update_job(job=job, update_mask=update_mask) # type: ignore
|
|
82
|
+
else:
|
|
83
|
+
job = client.create_job(parent=parent, job=job)
|
|
84
|
+
|
|
85
|
+
if job_name != task.cloud_scheduler_job_name:
|
|
86
|
+
task.cloud_scheduler_job_name = job_name
|
|
87
|
+
task.save(update_fields=["cloud_scheduler_job_name"])
|
|
88
|
+
if job.state != job.State.ENABLED and task.state == task.State.ENABLED:
|
|
89
|
+
client.resume_job(name=job_name)
|
|
90
|
+
if job.state != job.State.DISABLED and task.state == task.State.DISABLED:
|
|
91
|
+
client.pause_job(name=job_name)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def delete_remote_scheduled_task(task: ScheduledTask):
|
|
95
|
+
from google.api_core.exceptions import NotFound
|
|
96
|
+
from google.cloud import scheduler_v1
|
|
97
|
+
|
|
98
|
+
client = scheduler_v1.CloudSchedulerClient()
|
|
99
|
+
if task.cloud_scheduler_job_name:
|
|
100
|
+
try:
|
|
101
|
+
client.delete_job(name=task.cloud_scheduler_job_name)
|
|
102
|
+
except NotFound:
|
|
103
|
+
pass
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from json import JSONDecodeError
|
|
4
|
+
|
|
5
|
+
from django.db import transaction
|
|
6
|
+
from django.http import JsonResponse
|
|
7
|
+
from django.shortcuts import get_object_or_404
|
|
8
|
+
from django.tasks import task_backends, InvalidTaskBackend
|
|
9
|
+
from django.views.decorators.csrf import csrf_exempt
|
|
10
|
+
from django.views.decorators.http import require_POST
|
|
11
|
+
|
|
12
|
+
from django_tasks_google.executor import execute_task
|
|
13
|
+
from django_tasks_google.models import TaskExecution, ScheduledTask
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger("django_tasks_google")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def handle_oidc_auth(request, audience, email):
|
|
20
|
+
from google.auth.transport import requests as google_requests
|
|
21
|
+
from google.oauth2 import id_token
|
|
22
|
+
|
|
23
|
+
auth_header = request.headers.get("Authorization", "")
|
|
24
|
+
if not auth_header.startswith("Bearer "):
|
|
25
|
+
return False, "Missing or invalid Authorization header"
|
|
26
|
+
|
|
27
|
+
token = auth_header[7:]
|
|
28
|
+
try:
|
|
29
|
+
claims = id_token.verify_oauth2_token(
|
|
30
|
+
token, google_requests.Request(), audience=audience
|
|
31
|
+
)
|
|
32
|
+
if claims.get("email") != email:
|
|
33
|
+
return False, f"Unexpected caller email: {claims.get('email')}"
|
|
34
|
+
if claims.get("email_verified") is not True:
|
|
35
|
+
return False, "Caller email is not verified"
|
|
36
|
+
return True, None
|
|
37
|
+
except Exception as e:
|
|
38
|
+
logger.error(f"OIDC token verification failed: {e}")
|
|
39
|
+
return False, str(e)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@require_POST
|
|
43
|
+
@csrf_exempt
|
|
44
|
+
@transaction.non_atomic_requests
|
|
45
|
+
def execute_task_view(request):
|
|
46
|
+
try:
|
|
47
|
+
data = json.loads(request.body)
|
|
48
|
+
backend = task_backends[data["backend"]]
|
|
49
|
+
except (JSONDecodeError, KeyError, InvalidTaskBackend) as e:
|
|
50
|
+
logger.warning(f"Cannot verify backend: {e}")
|
|
51
|
+
return JsonResponse(
|
|
52
|
+
{
|
|
53
|
+
"error": "Unauthorized",
|
|
54
|
+
"detail": f"Cannot verify backend: {request.body}",
|
|
55
|
+
},
|
|
56
|
+
status=401,
|
|
57
|
+
)
|
|
58
|
+
is_valid, error_message = handle_oidc_auth(
|
|
59
|
+
request,
|
|
60
|
+
audience=backend.target_url,
|
|
61
|
+
email=backend.oidc_service_account,
|
|
62
|
+
)
|
|
63
|
+
if not is_valid:
|
|
64
|
+
logger.warning(f"Authentication failed: {error_message}")
|
|
65
|
+
return JsonResponse(
|
|
66
|
+
{"error": "Unauthorized", "detail": error_message},
|
|
67
|
+
status=401,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
if request.headers.get("X-CloudScheduler"):
|
|
71
|
+
job_name = request.headers["X-CloudScheduler-JobName"]
|
|
72
|
+
schedule_time = request.headers["X-CloudScheduler-ScheduleTime"]
|
|
73
|
+
cloud_scheduler_idempotency_key = f"{job_name}:{schedule_time}"
|
|
74
|
+
execution = TaskExecution.objects.filter(
|
|
75
|
+
cloud_scheduler_idempotency_key=cloud_scheduler_idempotency_key,
|
|
76
|
+
).first()
|
|
77
|
+
if not execution:
|
|
78
|
+
task = get_object_or_404(
|
|
79
|
+
ScheduledTask, name=job_name, backend=backend.alias
|
|
80
|
+
)
|
|
81
|
+
execution = TaskExecution.objects.create(
|
|
82
|
+
module_path=task.module_path,
|
|
83
|
+
backend=task.backend,
|
|
84
|
+
queue_name=task.name,
|
|
85
|
+
takes_context=task.takes_context,
|
|
86
|
+
args=task.args,
|
|
87
|
+
kwargs=task.kwargs,
|
|
88
|
+
cloud_scheduler_idempotency_key=cloud_scheduler_idempotency_key,
|
|
89
|
+
)
|
|
90
|
+
else:
|
|
91
|
+
execution = get_object_or_404(
|
|
92
|
+
TaskExecution, pk=data["task_execution_id"], backend=backend.alias
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
ok = execute_task(execution)
|
|
96
|
+
return JsonResponse(
|
|
97
|
+
{
|
|
98
|
+
"id": execution.pk,
|
|
99
|
+
"status": execution.status.value,
|
|
100
|
+
"errors": execution.errors,
|
|
101
|
+
"args": execution.args,
|
|
102
|
+
"kwargs": execution.kwargs,
|
|
103
|
+
"return_value": execution.return_value,
|
|
104
|
+
},
|
|
105
|
+
status=200 if ok else 500,
|
|
106
|
+
)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-tasks-google
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Django's Task Framework backends for: Cloud Tasks, Cloud Scheduler, and Cloud Run Jobs
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: django>=6.0.0
|
|
8
|
+
Requires-Dist: google-cloud-run>=0.1.0
|
|
9
|
+
Requires-Dist: google-cloud-scheduler>=2.0.0
|
|
10
|
+
Requires-Dist: google-cloud-tasks>=2.0.0
|
|
11
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
django_tasks_google/__init__.py
|
|
5
|
+
django_tasks_google/admin.py
|
|
6
|
+
django_tasks_google/backends.py
|
|
7
|
+
django_tasks_google/executor.py
|
|
8
|
+
django_tasks_google/models.py
|
|
9
|
+
django_tasks_google/scheduler.py
|
|
10
|
+
django_tasks_google/tasks.py
|
|
11
|
+
django_tasks_google/urls.py
|
|
12
|
+
django_tasks_google/views.py
|
|
13
|
+
django_tasks_google.egg-info/PKG-INFO
|
|
14
|
+
django_tasks_google.egg-info/SOURCES.txt
|
|
15
|
+
django_tasks_google.egg-info/dependency_links.txt
|
|
16
|
+
django_tasks_google.egg-info/requires.txt
|
|
17
|
+
django_tasks_google.egg-info/top_level.txt
|
|
18
|
+
django_tasks_google/management/__init__.py
|
|
19
|
+
django_tasks_google/management/commands/__init__.py
|
|
20
|
+
django_tasks_google/management/commands/execute_task.py
|
|
21
|
+
django_tasks_google/management/commands/sync_scheduled_tasks.py
|
|
22
|
+
django_tasks_google/migrations/0001_initial.py
|
|
23
|
+
django_tasks_google/migrations/__init__.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
django_tasks_google
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "django-tasks-google"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Django's Task Framework backends for: Cloud Tasks, Cloud Scheduler, and Cloud Run Jobs"
|
|
5
|
+
requires-python = ">=3.12"
|
|
6
|
+
dependencies = [
|
|
7
|
+
"django>=6.0.0",
|
|
8
|
+
"google-cloud-run>=0.1.0",
|
|
9
|
+
"google-cloud-scheduler>=2.0.0",
|
|
10
|
+
"google-cloud-tasks>=2.0.0",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
[dependency-groups]
|
|
14
|
+
dev = [
|
|
15
|
+
"ruff>=0.15.7",
|
|
16
|
+
]
|