django-maintenance-tasks 0.1.0__tar.gz

django_maintenance_tasks-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,44 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - run: pip install -e ".[dev]"
+      - run: pytest
+
+  build:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - run: pip install build
+      - run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # trusted publishing, no API token needed
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

django_maintenance_tasks-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.pytest_cache/

django_maintenance_tasks-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ajinkya Pisal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

django_maintenance_tasks-0.1.0/PKG-INFO

@@ -0,0 +1,176 @@
+Metadata-Version: 2.5
+Name: django-maintenance-tasks
+Version: 0.1.0
+Summary: Pausable, resumable, throttled data backfills for Django, built on django.tasks.
+Project-URL: Homepage, https://github.com/ajinkyapisal/django-maintenance-tasks
+Project-URL: Issues, https://github.com/ajinkyapisal/django-maintenance-tasks/issues
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 6.0
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.12
+Requires-Dist: django>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-django; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# django-maintenance-tasks
+
+Pausable, resumable, throttled data backfills for Django, inspired by Shopify's
+[`maintenance_tasks`](https://github.com/Shopify/maintenance_tasks) for Rails.
+
+Backfilling millions of rows from a management command or a `RunPython`
+migration is risky: it can lock tables, slow the site down, and if it crashes
+halfway it starts again from zero. This package turns each backfill into a small
+class that runs in the background in batches, saves its position as it goes, and
+can be started, paused, resumed and cancelled from the Django admin.
+
+It runs on Django's built-in [Tasks framework](https://docs.djangoproject.com/en/stable/topics/tasks/),
+so it works with any task backend: the database backend from `django-tasks`, RQ,
+Celery and others.
+
+> Status: early scaffold. The API may change.
+
+## Install
+
+Requires Python 3.12+ and Django 6.0+. Not on PyPI yet, so install from GitHub:
+
+```bash
+pip install git+https://github.com/ajinkyapisal/django-maintenance-tasks
+```
+
+```python
+INSTALLED_APPS = [
+    # ...
+    "maintenance_tasks",
+]
+
+# Any django.tasks backend that runs tasks in a worker.
+TASKS = {"default": {"BACKEND": "..."}}
+```
+
+```bash
+python manage.py migrate
+```
+
+## Write a task
+
+Put tasks in `<app>/maintenance_tasks.py`. They are discovered automatically.
+
+```python
+from django.utils.text import slugify
+
+from maintenance_tasks import MaintenanceTask
+from blog.models import Post
+
+
+class BackfillSlugs(MaintenanceTask):
+    description = "Fill in missing post slugs."
+    batch_size = 500
+
+    def collection(self):
+        return Post.objects.filter(slug="")
+
+    def process(self, post):
+        post.slug = slugify(post.title)
+        post.save(update_fields=["slug"])
+```
+
+- `collection()` returns a QuerySet, a list or another iterable. QuerySets are
+  walked in primary key order. Other iterables are walked by position, so they
+  must return the same items in the same order every time.
+- `process(item)` handles one item. It must be safe to run twice on the same
+  item: after a crash, up to one batch can be processed again.
+- `self.arguments` holds the JSON arguments the run was started with.
+- `count()` gives the total for the progress bar. Override it if counting is
+  slow.
+- `throttle()` returns `True` to back off for `throttle_backoff` seconds, for
+  example when database replicas are lagging.
+
+## Run it
+
+From the admin: **Maintenance tasks → Runs → Start a run**. Select runs to
+pause, resume or cancel them. Starting and controlling runs needs the
+`maintenance_tasks.add_run` permission.
+
+From the command line:
+
+```bash
+python manage.py maintenance_tasks list
+python manage.py maintenance_tasks run blog.maintenance_tasks.BackfillSlugs --arguments '{"batch": "2024-q1"}'
+```
+
+From code:
+
+```python
+from maintenance_tasks import runner
+
+run = runner.start("blog.maintenance_tasks.BackfillSlugs")
+runner.pause(run)
+runner.resume(run)
+runner.cancel(run)
+```
+
+## How it works
+
+- Each run is a `Run` row with a status, a cursor and progress counts. Only one
+  active run per task is allowed.
+- A run is processed by a `django.tasks` job. After every batch it saves the
+  cursor and checks whether a pause or cancel was requested.
+- A job gives up its worker after `MAX_RUNTIME` seconds (default 120) and
+  enqueues the next job, so long backfills never block a worker for hours:
+
+  ```python
+  MAINTENANCE_TASKS = {"MAX_RUNTIME": 120, "STALE_AFTER": 600}
+  ```
+
+- If a worker dies mid-run (killed, out of memory, machine lost), the run stays
+  `running` with an old heartbeat. `recover_stalled()` puts it back on the
+  queue from its last checkpoint. Run it from cron, or use the admin action:
+
+  ```bash
+  python manage.py maintenance_tasks recover
+  ```
+
+  A run counts as stalled after `STALE_AFTER` seconds without a heartbeat
+  (default 600). Keep it well above the time one batch takes.
+- Every job carries a generation number. Recovering a run starts a new
+  generation, so if the old worker was only frozen and comes back, it stops at
+  its next checkpoint instead of running alongside the new one. Duplicate
+  deliveries of the same job are ignored too.
+- If `process()` raises, the run is marked `errored` with the exception and
+  traceback, and the cursor points at the last item that succeeded.
+- Throttle backoff uses `run_after` when the backend supports deferred tasks.
+  Otherwise the worker sleeps for the backoff time.
+
+## Roadmap
+
+- CSV upload as a collection
+- Typed task parameters with a generated admin form
+- Dry runs
+- Retrying errored runs from the cursor
+- Live progress in the admin
+- Run history: who started what, with which arguments
+
+## Development
+
+```bash
+python3.13 -m venv .venv
+.venv/bin/pip install -e ".[dev]"
+.venv/bin/pytest
+```
+
+`example/` is a demo project on Postgres with the `django-tasks-db` backend.
+`example/e2e.py` runs real `db_worker` processes and checks pausing,
+cancelling, throttling, errors, graceful shutdown, killed workers and frozen
+workers:
+
+```bash
+docker run -d --name dmt-pg -e POSTGRES_HOST_AUTH_METHOD=trust \
+    -e POSTGRES_DB=dmt_demo -p 55432:5432 postgres:17-alpine
+.venv/bin/pip install django-tasks-db "psycopg[binary]"
+.venv/bin/python example/manage.py migrate
+.venv/bin/python example/e2e.py
+```

django_maintenance_tasks-0.1.0/README.md

@@ -0,0 +1,158 @@
+# django-maintenance-tasks
+
+Pausable, resumable, throttled data backfills for Django, inspired by Shopify's
+[`maintenance_tasks`](https://github.com/Shopify/maintenance_tasks) for Rails.
+
+Backfilling millions of rows from a management command or a `RunPython`
+migration is risky: it can lock tables, slow the site down, and if it crashes
+halfway it starts again from zero. This package turns each backfill into a small
+class that runs in the background in batches, saves its position as it goes, and
+can be started, paused, resumed and cancelled from the Django admin.
+
+It runs on Django's built-in [Tasks framework](https://docs.djangoproject.com/en/stable/topics/tasks/),
+so it works with any task backend: the database backend from `django-tasks`, RQ,
+Celery and others.
+
+> Status: early scaffold. The API may change.
+
+## Install
+
+Requires Python 3.12+ and Django 6.0+. Not on PyPI yet, so install from GitHub:
+
+```bash
+pip install git+https://github.com/ajinkyapisal/django-maintenance-tasks
+```
+
+```python
+INSTALLED_APPS = [
+    # ...
+    "maintenance_tasks",
+]
+
+# Any django.tasks backend that runs tasks in a worker.
+TASKS = {"default": {"BACKEND": "..."}}
+```
+
+```bash
+python manage.py migrate
+```
+
+## Write a task
+
+Put tasks in `<app>/maintenance_tasks.py`. They are discovered automatically.
+
+```python
+from django.utils.text import slugify
+
+from maintenance_tasks import MaintenanceTask
+from blog.models import Post
+
+
+class BackfillSlugs(MaintenanceTask):
+    description = "Fill in missing post slugs."
+    batch_size = 500
+
+    def collection(self):
+        return Post.objects.filter(slug="")
+
+    def process(self, post):
+        post.slug = slugify(post.title)
+        post.save(update_fields=["slug"])
+```
+
+- `collection()` returns a QuerySet, a list or another iterable. QuerySets are
+  walked in primary key order. Other iterables are walked by position, so they
+  must return the same items in the same order every time.
+- `process(item)` handles one item. It must be safe to run twice on the same
+  item: after a crash, up to one batch can be processed again.
+- `self.arguments` holds the JSON arguments the run was started with.
+- `count()` gives the total for the progress bar. Override it if counting is
+  slow.
+- `throttle()` returns `True` to back off for `throttle_backoff` seconds, for
+  example when database replicas are lagging.
+
+## Run it
+
+From the admin: **Maintenance tasks → Runs → Start a run**. Select runs to
+pause, resume or cancel them. Starting and controlling runs needs the
+`maintenance_tasks.add_run` permission.
+
+From the command line:
+
+```bash
+python manage.py maintenance_tasks list
+python manage.py maintenance_tasks run blog.maintenance_tasks.BackfillSlugs --arguments '{"batch": "2024-q1"}'
+```
+
+From code:
+
+```python
+from maintenance_tasks import runner
+
+run = runner.start("blog.maintenance_tasks.BackfillSlugs")
+runner.pause(run)
+runner.resume(run)
+runner.cancel(run)
+```
+
+## How it works
+
+- Each run is a `Run` row with a status, a cursor and progress counts. Only one
+  active run per task is allowed.
+- A run is processed by a `django.tasks` job. After every batch it saves the
+  cursor and checks whether a pause or cancel was requested.
+- A job gives up its worker after `MAX_RUNTIME` seconds (default 120) and
+  enqueues the next job, so long backfills never block a worker for hours:
+
+  ```python
+  MAINTENANCE_TASKS = {"MAX_RUNTIME": 120, "STALE_AFTER": 600}
+  ```
+
+- If a worker dies mid-run (killed, out of memory, machine lost), the run stays
+  `running` with an old heartbeat. `recover_stalled()` puts it back on the
+  queue from its last checkpoint. Run it from cron, or use the admin action:
+
+  ```bash
+  python manage.py maintenance_tasks recover
+  ```
+
+  A run counts as stalled after `STALE_AFTER` seconds without a heartbeat
+  (default 600). Keep it well above the time one batch takes.
+- Every job carries a generation number. Recovering a run starts a new
+  generation, so if the old worker was only frozen and comes back, it stops at
+  its next checkpoint instead of running alongside the new one. Duplicate
+  deliveries of the same job are ignored too.
+- If `process()` raises, the run is marked `errored` with the exception and
+  traceback, and the cursor points at the last item that succeeded.
+- Throttle backoff uses `run_after` when the backend supports deferred tasks.
+  Otherwise the worker sleeps for the backoff time.
+
+## Roadmap
+
+- CSV upload as a collection
+- Typed task parameters with a generated admin form
+- Dry runs
+- Retrying errored runs from the cursor
+- Live progress in the admin
+- Run history: who started what, with which arguments
+
+## Development
+
+```bash
+python3.13 -m venv .venv
+.venv/bin/pip install -e ".[dev]"
+.venv/bin/pytest
+```
+
+`example/` is a demo project on Postgres with the `django-tasks-db` backend.
+`example/e2e.py` runs real `db_worker` processes and checks pausing,
+cancelling, throttling, errors, graceful shutdown, killed workers and frozen
+workers:
+
+```bash
+docker run -d --name dmt-pg -e POSTGRES_HOST_AUTH_METHOD=trust \
+    -e POSTGRES_DB=dmt_demo -p 55432:5432 postgres:17-alpine
+.venv/bin/pip install django-tasks-db "psycopg[binary]"
+.venv/bin/python example/manage.py migrate
+.venv/bin/python example/e2e.py
+```

django_maintenance_tasks-0.1.0/example/blog/maintenance_tasks.py

@@ -0,0 +1,35 @@
+import time
+from pathlib import Path
+
+from django.utils.text import slugify
+
+from example.blog.models import Post
+from maintenance_tasks import MaintenanceTask
+
+
+class BackfillSlugs(MaintenanceTask):
+    """Fills in missing slugs.
+
+    Arguments:
+        delay: seconds to sleep per post, to make the run slow enough to watch.
+        throttle_file: while this file exists, the task backs off.
+        fail_at: raise when processing the post with this title.
+    """
+
+    description = "Fill in missing post slugs."
+    batch_size = 100
+    throttle_backoff = 2
+
+    def collection(self):
+        return Post.objects.filter(slug="")
+
+    def process(self, post):
+        if post.title == self.arguments.get("fail_at"):
+            raise ValueError(f"cannot slugify {post.title!r}")
+        time.sleep(self.arguments.get("delay", 0))
+        post.slug = slugify(post.title)
+        post.save(update_fields=["slug"])
+
+    def throttle(self):
+        path = self.arguments.get("throttle_file")
+        return bool(path) and Path(path).exists()

django_maintenance_tasks-0.1.0/example/blog/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 6.1.1 on 2026-10-03 08:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Post',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=200)),
+                ('slug', models.SlugField(blank=True)),
+            ],
+        ),
+    ]

django_maintenance_tasks-0.1.0/example/blog/models.py

@@ -0,0 +1,6 @@
+from django.db import models
+
+
+class Post(models.Model):
+    title = models.CharField(max_length=200)
+    slug = models.SlugField(blank=True)