django-pfx 1.7.6.dev7__tar.gz → 1.7.6.dev9__tar.gz

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/.gitlab-ci.yml

@@ -39,6 +39,7 @@ quality check:
     - python manage.py makemigrations tests_base_user --check --dry-run
     - python manage.py makemigrations tests_custom_user --check --dry-run
     - python manage.py makemigrations tests_user_email_login --check --dry-run
+    - python manage.py makemigrations tests_custom_user_job --check --dry-run
 
 test:
   image: $IMAGE
@@ -54,6 +55,7 @@ test:
     - coverage run -a --source='.' manage.py test tests_base_user
     - coverage run -a --source='.' manage.py test tests_custom_user
     - coverage run -a --source='.' manage.py test tests_user_email_login
+    - coverage run -a --source='.' manage.py test tests_custom_user_job
     - coverage report
     - coverage xml
     - coverage html

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: django-pfx
-Version: 1.7.6.dev7
+Version: 1.7.6.dev9
 Summary: Django PFX is a toolkit designed to streamline the development of RESTful APIs using the Django framework.
 Author: Hervé Martinet
 Author-email: herve.martinet@gmail.com

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/django_pfx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: django-pfx
-Version: 1.7.6.dev7
+Version: 1.7.6.dev9
 Summary: Django PFX is a toolkit designed to streamline the development of RESTful APIs using the Django framework.
 Author: Hervé Martinet
 Author-email: herve.martinet@gmail.com

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/django_pfx.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ doc/source/pfx_views.md
 doc/source/profiling.md
 doc/source/settings.md
 doc/source/testing.md
+doc/source/user_jobs.md
 img/pfx.png
 img/pfx.svg
 pfx/__init__.py
@@ -177,6 +178,7 @@ pfx/userjobs/migrations/0001_initial.py
 pfx/userjobs/migrations/0002_alter_userjob_options_alter_userjob_auto_queued_and_more.py
 pfx/userjobs/migrations/0003_clean_scheduled_tasks.py
 pfx/userjobs/migrations/0004_userjob_summary_url.py
+pfx/userjobs/migrations/0005_swappable_user_job.py
 pfx/userjobs/migrations/__init__.py
 pfx/userjobs/models/__init__.py
 pfx/userjobs/models/user_job.py
@@ -276,6 +278,21 @@ tests_custom_user/settings/dev_default.py
 tests_custom_user/tests/__init__.py
 tests_custom_user/tests/test_api.py
 tests_custom_user/tests/test_auth_api.py
+tests_custom_user_job/__init__.py
+tests_custom_user_job/models.py
+tests_custom_user_job/providers.py
+tests_custom_user_job/scope.py
+tests_custom_user_job/urls.py
+tests_custom_user_job/migrations/0001_initial.py
+tests_custom_user_job/migrations/__init__.py
+tests_custom_user_job/settings/__init__.py
+tests_custom_user_job/settings/ci.py
+tests_custom_user_job/settings/common.py
+tests_custom_user_job/settings/dev.py
+tests_custom_user_job/settings/dev_custom_example.py
+tests_custom_user_job/settings/dev_default.py
+tests_custom_user_job/tests/__init__.py
+tests_custom_user_job/tests/test_user_job.py
 tests_user_email_login/__init__.py
 tests_user_email_login/models.py
 tests_user_email_login/urls.py

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/django_pfx.egg-info/scm_file_list.json

@@ -20,6 +20,7 @@
     "doc/source/profiling.md",
     "doc/source/settings.md",
     "doc/source/testing.md",
+    "doc/source/user_jobs.md",
     "img/pfx.png",
     "img/pfx.svg",
     "make_messages",
@@ -163,6 +164,7 @@
     "pfx/userjobs/migrations/0002_alter_userjob_options_alter_userjob_auto_queued_and_more.py",
     "pfx/userjobs/migrations/0003_clean_scheduled_tasks.py",
     "pfx/userjobs/migrations/0004_userjob_summary_url.py",
+    "pfx/userjobs/migrations/0005_swappable_user_job.py",
     "pfx/userjobs/migrations/__init__.py",
     "pfx/userjobs/models/__init__.py",
     "pfx/userjobs/models/user_job.py",
@@ -269,6 +271,21 @@
     "tests_custom_user/tests/test_auth_api.py",
     "tests_custom_user/urls.py",
     "tests_custom_user/views.py",
+    "tests_custom_user_job/__init__.py",
+    "tests_custom_user_job/migrations/0001_initial.py",
+    "tests_custom_user_job/migrations/__init__.py",
+    "tests_custom_user_job/models.py",
+    "tests_custom_user_job/providers.py",
+    "tests_custom_user_job/scope.py",
+    "tests_custom_user_job/settings/__init__.py",
+    "tests_custom_user_job/settings/ci.py",
+    "tests_custom_user_job/settings/common.py",
+    "tests_custom_user_job/settings/dev.py",
+    "tests_custom_user_job/settings/dev_custom_example.py",
+    "tests_custom_user_job/settings/dev_default.py",
+    "tests_custom_user_job/tests/__init__.py",
+    "tests_custom_user_job/tests/test_user_job.py",
+    "tests_custom_user_job/urls.py",
     "tests_user_email_login/__init__.py",
     "tests_user_email_login/migrations/0001_initial.py",
     "tests_user_email_login/migrations/__init__.py",

django_pfx-1.7.6.dev9/django_pfx.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "1.7.5",
+  "distance": 9,
+  "node": "g7626d2f586fb8dbaa121ce3d2676a0e0c185f7fe",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-09-01"
+}

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/doc/index.rst

@@ -21,6 +21,7 @@ Topics
    source/model
    source/internationalisation
    source/authentication
+   source/user_jobs
    source/settings
    source/testing
    source/profiling

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/doc/source/settings.md

@@ -93,3 +93,40 @@ Refer to [MDN documentation](https://developer.mozilla.org/en-US/docs/Web/HTTP/H
 
 ### PFX_FORCE_USER_FILTERED_QUERYSET
 Force the use of a specific queryset that is segregated by user.
+
+## User jobs
+
+See [User jobs](./user_jobs.md).
+
+### PFX_USER_JOB_MODEL
+The model used for user jobs, as an `app_label.ModelName` reference
+(optional, default `userjobs.UserJob`).
+
+It must inherit `pfx.userjobs.models.AbstractUserJob`, see
+[Extending the model](./user_jobs.md#extending-the-model).
+
+### PFX_USER_JOB_EXECUTOR
+The class queuing the jobs and registering the periodic tasks
+(optional, default `pfx.userjobs.providers.UserJobDjangoQExecutor`).
+
+### PFX_USER_JOB_WEBSOCKET_PROVIDER
+The class sending the job status changes to the clients
+(optional, default
+`pfx.userjobs.providers.UserJobWebsocketChannelProvider`).
+
+Override it to narrow the audience of a job, see
+[Providers](./user_jobs.md#providers).
+
+### PFX_USER_JOB_TIMEOUT_SUMMARY
+The summary given to a job failed for exceeding its timeout
+(optional, default *Timeout exceeds. Our support team has been
+notified.*).
+
+### PFX_USER_JOB_NEVER_STARTED_SUMMARY
+The summary given to a job failed for never having started
+(optional, default *This job never started. Our support team has been
+notified.*).
+
+### USER_JOB_FAILED_SHELF_LIFE_HOURS
+How long a failed job is kept, in hours, unless its own `shelf_life` is
+longer (optional, default `24`).

django_pfx-1.7.6.dev9/doc/source/user_jobs.md

@@ -0,0 +1,401 @@
+# User jobs
+
+A user job is a long task run out of the request/response cycle, on
+behalf of the user who started it: an export, a mailing, a document
+generation. The API answers immediately with the job, the task runs in a
+worker, and its progress is pushed back to the browser over a websocket.
+
+PFX keeps the job itself: its status, its result summary, the files it
+produced, and how long all of that is kept.
+
+## Setup
+
+The app needs a queue and a channel layer:
+
+```bash
+pip install django-q2
+pip install channels[daphne] channels-redis
+```
+
+```python
+# settings.py
+INSTALLED_APPS = [
+    ...
+    'pfx.userjobs',
+    'django_q',
+]
+```
+
+Both are checked when the app is loaded, and are replaceable, see
+[Providers](#providers).
+
+```python
+# urls.py
+urlpatterns = [
+    path('api/', include('pfx.userjobs.urls')),
+]
+```
+
+```python
+# asgi.py
+from pfx.userjobs.urls import websocket_urlpatterns as userjobs_ws
+```
+
+## Writing a job
+
+A job is a module level function taking the job as first argument,
+followed by the arguments it was created with:
+
+```python
+def export_books(job, author_pk):
+    books = Book.objects.filter(author_id=author_pk)
+    return dict(summary=f"{books.count()} books exported.")
+```
+
+It is found by its dotted path, so it must be importable and stay where
+it is as long as jobs referencing it may still be queued.
+
+### The result
+
+The returned dictionary describes what the user is shown:
+
+| Key | |
+| --- | --- |
+| `status` | `done` (the default), `warning` or `failed` |
+| `summary` | one line, shown as the result of the job |
+| `summary_url` | where the summary links to |
+| `message` | a longer text, shown when the job is opened |
+| `stacktrace` | technical details, for a failure |
+
+An exception that escapes the function fails the job: the summary
+becomes `settings.PFX_SYSTEM_ERROR_USER_MESSAGE` and the traceback is
+kept in `stacktrace`. Nothing is shown to the user that the function did
+not choose to show.
+
+`failed()` reports a failure the function handles itself:
+
+```python
+def export_books(job, author_pk):
+    try:
+        ...
+    except SourceUnavailable:
+        job.failed(summary="The catalog is unavailable, try again later.")
+```
+
+### Attachments
+
+A job produces files through `UserJobAttachment`. They are listed in the
+job and downloaded from the job REST API:
+
+```python
+from pfx.userjobs.models import UserJobAttachment
+
+
+def export_books(job, author_pk):
+    attachment = UserJobAttachment.objects.create(job=job, is_result=True)
+    attachment.file = UserJobAttachment._meta.get_field('file').upload(
+        attachment, pdf_bytes, "books.pdf")
+    attachment.save()
+    return dict(summary="The export is ready.")
+```
+
+They are deleted with the job, files included.
+
+### Jobs that outlive the worker
+
+A function returning `status='started'` leaves the job running: PFX does
+not terminate it. Use it for a task handed over to a third party, and
+call `terminated()` or `failed()` from wherever the answer comes back:
+
+```python
+def send_campaign(job, campaign_pk):
+    provider.send(campaign_pk, callback_job=job.pk)
+    return dict(status=job.STATUS_STARTED)
+```
+
+Such a job is still failed after its `timeout`, see
+[Life cycle](#life-cycle).
+
+## Starting a job
+
+```python
+from pfx.userjobs.models import get_user_job_model
+
+
+class BookRestView(RestView):
+    @rest_api("/<int:pk>/export", method="get")
+    def export(self, pk, *args, **kwargs):
+        job = get_user_job_model().objects.create(
+            name="Books export",
+            user=self.request.user,
+            func='myapp.jobs.export_books',
+            args=[pk],
+            timeout=300)
+        return JsonResponse(dict(job=job.pk))
+```
+
+| Field | |
+| --- | --- |
+| `name` | what the user sees in their job list |
+| `user` | who started it, and who may see it |
+| `func` | dotted path of the function |
+| `args` | its arguments, stored as JSON |
+| `timeout` | in seconds, given to the queue |
+| `cluster` | the queue cluster, `default` |
+| `shelf_life` | how long the finished job is kept, 30 days |
+| `lock` | see [One job at a time](#one-job-at-a-time) |
+| `auto_queued` | queue the job on save, `True` |
+
+Creating the job queues it. With `auto_queued=False` it is created idle
+and queued later by `enqueue()`, which is what a job created outside of
+a request usually wants.
+
+Always go through `get_user_job_model()` rather than importing `UserJob`:
+the model is swappable, see [Extending the model](#extending-the-model).
+
+### One job at a time
+
+A non empty `lock` is unique among the jobs that are not finished: as
+long as one is queued or running, creating another with the same lock
+raises an `APIError`, answered as *This operation is already in
+progress.*
+
+```python
+get_user_job_model().objects.create(
+    ...,
+    lock=f'export-books-{author.pk}')
+```
+
+## Life cycle
+
+```{mermaid}
+
+   stateDiagram-v2
+      [*] --> not_queued: created
+      not_queued --> queued: enqueue()
+      queued --> started: the worker picks it up
+      started --> done: the function returns
+      started --> warning: status='warning'
+      started --> failed: status='failed', or an exception
+      done --> [*]: shelf life
+      warning --> [*]: shelf life
+      failed --> [*]: shelf life
+
+```
+
+Two periodic tasks watch over the jobs, registered on the queue after
+each `migrate`:
+
+`pfx.userjobs.tasks.task_user_jobs_update`, every 30 minutes, closes the
+jobs nothing else will:
+
+- a job whose queue task failed,
+- a job still `started` more than `timeout` + 5 minutes after it started,
+  failed with `PFX_USER_JOB_TIMEOUT_SUMMARY`,
+- a job still `queued` after 3 days, failed with
+  `PFX_USER_JOB_NEVER_STARTED_SUMMARY`,
+- a job still `not_queued` after 3 days, deleted.
+
+`pfx.userjobs.tasks.task_user_jobs_cleanup`, every night, deletes the
+finished jobs past their retention, see [Retention](#retention).
+
+## Retention
+
+A job is kept for its `shelf_life` after it ended, 30 days by default. A
+negative `shelf_life` keeps it forever.
+
+Failed jobs are kept for `USER_JOB_FAILED_SHELF_LIFE_HOURS` hours, 24 by
+default, unless their own `shelf_life` is longer: a failure is of no use
+to the user once they have read it, but a job worth keeping stays.
+
+Deleting a job deletes its attachments and their files.
+
+## The REST API
+
+`/user-jobs/user-jobs` lists the jobs of the authenticated user, most
+recent first. A user only ever sees their own jobs.
+
+| | |
+| --- | --- |
+| `GET /user-jobs/user-jobs` | the list |
+| `GET /user-jobs/user-jobs/<pk>` | one job and its attachments |
+| `PUT /user-jobs/user-jobs/<pk>` | mark as read |
+| `DELETE /user-jobs/user-jobs/<pk>` | delete a finished job |
+| `GET /user-jobs/attachments/<pk>/file` | download an attachment |
+
+Everything is read only but `read_status`: writing the current status of
+the job into it marks the job as read. The `unread` filter returns the
+jobs whose `read_status` does not match their `status` yet, which is what
+a notification badge counts.
+
+The list also filters on `status`, on `status_exclude` and on
+`created_after`.
+
+A job can only be deleted once it is `done`, `warning` or `failed`, a
+superuser excepted.
+
+## The websocket
+
+The client connects to `ws/user` and receives a message on every status
+change of its jobs:
+
+```json
+{
+    "model": "base.UserJob",
+    "event": "status_changed",
+    "item": {"pk": 12, "status": {"value": "done", "label": "Completed"}}
+}
+```
+
+`event` is `status_changed` or `deleted`, and `item` carries the job as
+the REST API returns it, with its attachments.
+
+`model` discriminates the message among the producers sharing the
+channel. Despite what it looks like it is a wire protocol value, not a
+model reference: it stays `base.UserJob` whatever
+`PFX_USER_JOB_MODEL` is, because `react-pfx` matches it as is. Making it
+follow the model would silently stop the clients from showing
+anything.
+
+The message is sent to the channel group of the user, and the client
+subscribes to the same group when it connects. Both names come from the
+websocket provider, which is where a project narrows the audience of a
+job, see [Providers](#providers).
+
+## Extending the model
+
+`UserJob` knows nothing about the domain of a project, and that is often
+not enough: a multi-tenant project segregating its data by tenant needs
+the jobs to be segregated too, otherwise a user working on several
+tenants sees, on one of them, the results of the jobs they started on
+another.
+
+The model is swappable. `AbstractUserJob` carries every field and the
+whole machinery, and the replacement inherits it: fields, constraints,
+managers and behaviour can be added, nothing has to be rewritten.
+
+```python
+# myapp/models.py
+from pfx.userjobs.models import AbstractUserJob
+
+
+class UserJobQuerySet(TenantQuerySetMixin, PFXUserJobQuerySet):
+    pass
+
+
+class UserJob(AbstractUserJob):
+    tenant = models.ForeignKey(Tenant, on_delete=models.PROTECT)
+
+    objects = UserJobQuerySet.as_manager()
+
+    class Meta(AbstractUserJob.Meta):
+        db_table = 'myapp_user_job'
+
+    def save(self, *args, **kw):
+        if self.tenant_id is None:
+            self.tenant = current_tenant()
+        return super().save(*args, **kw)
+```
+
+```python
+# settings.py
+PFX_USER_JOB_MODEL = 'myapp.UserJob'
+```
+
+The setting takes an `app_label.ModelName`, the Django convention for a
+swappable model, as `AUTH_USER_MODEL`.
+
+`objects` is the manager the project narrows, and the one the rest views
+and the attachments go through. `all_objects`, inherited from
+`AbstractUserJob`, must stay unnarrowed: the worker running a job and
+the periodic tasks use it, and they run outside of any request.
+
+### The migration
+
+`userjobs.0005_swappable_user_job` repoints the attachments to the
+swapped model, so it has to run after the migration creating it. PFX
+depends on the *first* migration of its app, which is not enough as soon
+as the model is added later: the project migration declares the
+ordering.
+
+```python
+class Migration(migrations.Migration):
+    dependencies = [('userjobs', '0004_userjob_summary_url')]
+    run_before = [('userjobs', '0005_swappable_user_job')]
+```
+
+Nothing else is needed: the foreign key of the attachments follows the
+setting, PFX alters it itself.
+
+### Taking over the PFX table
+
+Swapping the model leaves the `userjobs_user_job` table behind, with the
+jobs it holds and the attachments pointing at them. PFX drops it if it
+is empty; a project with jobs to keep takes it over in the migration
+above, instead of creating a table of its own. Renaming is enough, the
+columns are the ones of `AbstractUserJob`:
+
+```python
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            state_operations=[
+                migrations.CreateModel(name='UserJob', ...)],
+            database_operations=[migrations.RunSQL(
+                "ALTER TABLE userjobs_user_job RENAME TO myapp_user_job;",
+                "ALTER TABLE myapp_user_job RENAME TO userjobs_user_job;")]),
+        migrations.AddField(model_name='userjob', name='tenant', ...),
+    ]
+```
+
+A field the jobs cannot do without is added nullable, filled from what
+is known of the existing jobs, then made mandatory.
+
+`ALTER TABLE ... RENAME` leaves the indexes named after the old table.
+Only the unique constraint on `lock` has to follow, Django knows it by
+name:
+
+```sql
+ALTER INDEX unique_active_lock
+    RENAME TO myapp_userjob_unique_active_lock;
+```
+
+### What the project still owns
+
+The swap is invisible to PFX, which always goes through
+`get_user_job_model()`, but not to the project: a
+`from pfx.userjobs.models import UserJob` left anywhere in it now points
+at the swapped out model, and raises *Manager isn't available* the
+first time it is used.
+
+## Providers
+
+Two classes are replaceable, both validated when the app is loaded.
+
+`PFX_USER_JOB_EXECUTOR` queues the jobs and registers the periodic
+tasks. The default one uses Django Q2.
+
+`PFX_USER_JOB_WEBSOCKET_PROVIDER` sends the status changes. Its two
+group names are what a project overrides to narrow the audience of a
+job, and they must stay consistent with each other:
+
+```python
+class TenantUserJobWebsocketChannelProvider(UserJobWebsocketChannelProvider):
+    @classmethod
+    def group_name(cls, job):
+        """The group notified of a status change."""
+        return f'group_user_{job.user.pk}_tenant_{job.tenant_id}'
+
+    @classmethod
+    def consumer_group_name(cls, consumer):
+        """The group a client subscribes to."""
+        return (
+            f'group_user_{consumer.user.pk}'
+            f'_tenant_{consumer.scope["tenant"].pk}')
+```
+
+The consumer is passed whole so that an override can use its scope, set
+by the websocket middlewares, and not only its user.
+
+## Settings
+
+See [Settings](./settings.md#user-jobs).

{django_pfx-1.7.6.dev7 → django_pfx-1.7.6.dev9}/pfx/settings/dev.py

@@ -8,6 +8,10 @@ INSTALLED_APPS = [
     'pfx.emails',
 ]
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+# The swappable user job model must be resolvable by the migration
+# autodetector.
+PFX_USER_JOB_MODEL = 'userjobs.UserJob'
 STORAGE_DEFAULT = 'pfx.pfxcore.storage.LocalStorage'
 
 CACHE_MEMORY = {