django-admin-background-task 0.4.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

bgtask/model_admin.py

@@ -103,7 +103,7 @@ class BGTaskModelAdmin(admin.ModelAdmin):
         if not task_name_to_desc:
             return BackgroundTask.objects.none()
 
-        bgts = list(
+        bgts = (
             BackgroundTask.objects.filter(
                 name__in=task_name_to_desc, namespace=self._bgtask_namespace
             )
@@ -123,6 +123,7 @@ class BGTaskModelAdmin(admin.ModelAdmin):
             )
             .order_by("-started_at", "-queued_at")
         )
+        bgts.add_position_in_queue()
         for bgt in bgts:
             bgt.admin_description = task_name_to_desc[bgt.name]
 

bgtask/models.py

@@ -1,4 +1,4 @@
-import functools
+import collections
 import logging
 import os
 import time
@@ -8,61 +8,52 @@ from contextlib import contextmanager
 
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
-from django.db import models, transaction
+from django.db import models
 from django.forms.models import model_to_dict
 from django.utils import timezone
 
 from model_utils import Choices
 
-
-log = logging.getLogger(__name__)
+from .utils import locked, only_if_state, q_or
 
 
-def locked(meth):
-    @functools.wraps(meth)
-    def _locked_meth(self, *args, **kwargs):
-        if getattr(self, "_locked", False):
-            return meth(self, *args, **kwargs)
+log = logging.getLogger(__name__)
 
-        with transaction.atomic():
-            BackgroundTask.objects.filter(id=self.id).select_for_update().only("id").get()
-            self.refresh_from_db()
 
-            # Mark as locked in case we are called recursively
-            self._locked = True
-            try:
-                return meth(self, *args, **kwargs)
-            finally:
-                self._locked = False
+class BackgroundTaskQuerySet(models.QuerySet):
+    def add_position_in_queue(self):
+        """This evaluates the queryset and adds position_in_queue to each one (which requires
+        more DB queries).
+        """
+        recent_unqueued_task_by_nsn = {(task.namespace, task.name): None for task in self}
+        for ns, name in recent_unqueued_task_by_nsn:
+            recent_unqueued_task_by_nsn[(ns, name)] = BackgroundTask.most_recently_unqueued_task(
+                ns, name
+            )
 
-    return _locked_meth
+        queued_tasks_by_nsn = BackgroundTask.queued_tasks_in_order_by_nsn_like(self)
 
+        for task in self:
+            if task.state != task.STATES.queued:
+                continue
 
-def only_if_state(state):
-    def only_if_state_decorator(meth):
-        def only_if_state_wrapper(self, *args, **kwargs):
-            states = (state,) if isinstance(state, str) else tuple(state)
-            if self.state not in states:
-                raise RuntimeError(
-                    "%s cannot execute %s as in state %s not one of %s"
-                    % (self, meth.__name__, self.state, states)
-                )
-            return meth(self, *args, **kwargs)
+            unqueued_task = recent_unqueued_task_by_nsn[(task.namespace, task.name)]
+            if unqueued_task is not None and task.queued_at < unqueued_task.queued_at:
+                # More recently queued task has already been dispatched so we should be going...
+                task.position_in_queue = 0
+                continue
 
-        return only_if_state_wrapper
+            task.position_in_queue = 0
+            for queued_task in queued_tasks_by_nsn[(task.namespace, task.name)]:
+                if queued_task.queued_at >= task.queued_at:
+                    break
+                task.position_in_queue += 1
 
-    return only_if_state_decorator
+        return self
 
 
 class BackgroundTask(models.Model):
     id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4)
-    name = models.CharField(
-        max_length=1000,
-        help_text=(
-            "Name (or type) of this task, is not unique "
-            "per task instance but generally per task functionality"
-        ),
-    )
     namespace = models.CharField(
         max_length=1000,
         default="",
@@ -72,6 +63,13 @@ class BackgroundTask(models.Model):
             "entire codebase, allowing them to be shorter and human readable"
         ),
     )
+    name = models.CharField(
+        max_length=1000,
+        help_text=(
+            "Name (or type) of this task, is not unique "
+            "per task instance but generally per task functionality"
+        ),
+    )
 
     STATES = Choices("not_started", "queued", "running", "success", "partial_success", "failed")
     state = models.CharField(max_length=16, default=STATES.not_started, choices=STATES)
@@ -100,16 +98,22 @@ class BackgroundTask(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
 
+    objects = models.Manager.from_queryset(BackgroundTaskQuerySet)()
+
     class Meta:
         ordering = ["created", "id"]
 
+    # This needs to be added dynamically to model instances, and is done by
+    # BackgroundTaskQuerySet.add_position_in_queue()
+    position_in_queue = None
+
     @property
     def task_dict(self):
         task_dict = model_to_dict(self)
         return {
             "id": str(self.id),
             "updated": self.updated.isoformat(),
-            "position_in_queue": self.get_position_in_queue(),
+            "position_in_queue": self.position_in_queue,
             **task_dict,
         }
 
@@ -121,27 +125,32 @@ class BackgroundTask(models.Model):
     def incomplete(self):
         return self.state in [self.STATES.not_started, self.STATES.running]
 
-    def get_position_in_queue(self):
-        if self.state != self.STATES.queued:
-            return None
+    @classmethod
+    def most_recently_unqueued_task(cls, namespace, name):
+        return (
+            cls.objects.filter(queued_at__isnull=False, namespace=namespace, name=name)
+            .exclude(state=cls.STATES.not_started)
+            .exclude(state=cls.STATES.queued)
+            .order_by("-queued_at")
+            .first()
+        )
 
-        if (
-            BackgroundTask.objects.filter(
-                queued_at__gt=self.queued_at,
-            )
-            .exclude(state=self.STATES.not_started)
-            .exclude(state=self.STATES.queued)
-            .exists()
+    @classmethod
+    def queued_tasks_in_order_by_nsn_like(cls, tasks):
+        queued_by_nsn = collections.defaultdict(list)
+        nsns = {(task.namespace, task.name) for task in tasks}
+        for task in (
+            cls.objects.filter(queued_at__isnull=False, state=cls.STATES.queued)
+            .filter(q_or(models.Q(namespace=nsn[0], name=nsn[1]) for nsn in nsns))
+            .order_by("queued_at")
         ):
-            # More recent tasks have started, assume we're next (either that or we're stuck)
-            return 0
+            queued_by_nsn[(task.namespace, task.name)].append(task)
+        return queued_by_nsn
 
-        # Otherwise approximate queue position as the number of tasks that were queued before us
-        # and are still in the queue. So if there is nothing in front of us we are 0 in the queue.
-        assert self.queued_at
-        return BackgroundTask.objects.filter(
-            queued_at__lt=self.queued_at, state=self.STATES.queued
-        ).count()
+    def set_steps_to_complete(self, steps_to_complete):
+        self.steps_to_complete = steps_to_complete
+        self.steps_completed = 0
+        self.save()
 
     @contextmanager
     def runs_single_step(self):
@@ -161,6 +170,14 @@ class BackgroundTask(models.Model):
         else:
             self.succeed()
 
+    @contextmanager
+    def fails_if_exception(self):
+        try:
+            yield
+        except Exception as exc:
+            self.fail(exc)
+            raise
+
     @locked
     @only_if_state(STATES.not_started)
     def queue(self):
@@ -170,7 +187,13 @@ class BackgroundTask(models.Model):
         self.save()
 
     @locked
-    @only_if_state((STATES.not_started, STATES.queued))
+    @only_if_state(
+        (STATES.not_started, STATES.queued),
+        # Allow start() to be called while running so that if a task's subtasks are queued and
+        # asynchronous each can call .start() independently so the first one that is executed will
+        # start the task.
+        no_op_states=(STATES.running,),
+    )
     def start(self):
         log.info("Background Task starting: %s", self.id)
         self.state = self.STATES.running

bgtask/templates/bgtask/admin/change_list.html

@@ -12,7 +12,7 @@
   {% if not admin_bg_tasks|length %}
   <p class="help" style="font-style: italic;">No recent background tasks</p>
   {% else %}
-  <table>
+  <table style="max-height: 300px; display: block; overflow-y: scroll;">
     <thead>
       <tr><th>Task name</th><th>Started</th><th>Finished</th><th>Status</th></tr>
     </thead>

bgtask/tests/test_bgtask.py

@@ -1,7 +1,11 @@
 import pytest
 
+from django.utils import timezone
+
 from bgtask.models import BackgroundTask
 
+pytestmark = pytest.mark.django_db
+
 
 @pytest.fixture
 def a_task():
@@ -30,6 +34,37 @@ def test_bgtask_immediate_failure(a_task):
     assert a_task.state == BackgroundTask.STATES.failed
 
     assert len(a_task.errors) == 1
-    assert a_task.errors[0]["datetime"] == "1970-01-01T00:00:00+00:00"
+    assert a_task.errors[0]["datetime"] == a_task.completed_at.isoformat()
     assert a_task.errors[0]["error_message"] == "Some global failure"
-    assert "test_bgtask_immediate_failure" in a_task.errors[0]["trackeback"]
+    assert "test_bgtask_immediate_failure" in a_task.errors[0]["traceback"]
+
+
+def test_bgtask_start_again(a_task):
+    a_task.start()
+    import time
+    time.sleep(0.001)
+
+    curr_started_at = a_task.started_at
+    assert timezone.now() > curr_started_at
+
+    # Starting again is no-op
+    a_task.start()
+    assert a_task.started_at == curr_started_at
+
+
+def test_fails_if_exception(a_task):
+    a_task.start()
+
+    with pytest.raises(Exception):
+        with a_task.fails_if_exception():
+            raise Exception("fail!")
+
+    assert a_task.state == BackgroundTask.STATES.failed
+
+
+def test_set_steps(a_task):
+    a_task.steps_completed = 10
+    a_task.set_steps_to_complete(20)
+    a_task.refresh_from_db()
+    assert a_task.steps_to_complete == 20
+    assert a_task.steps_completed == 0

bgtask/utils.py

@@ -0,0 +1,56 @@
+import functools
+import operator
+from typing import Iterable
+
+from django.db import models, transaction
+
+
+# https://stackoverflow.com/questions/29900386/how-to-construct-django-q-object-matching-none
+Q_NONE = models.Q(pk__in=[])
+
+
+def q_or(q_objects: Iterable[models.Q]):
+    """Chain an iterable of Q with OR.
+
+    The more Q()s are passed, the more things are filtered in, so it
+    makes sense if nothing is passed nothing is filtered in.
+    """
+    return functools.reduce(operator.or_, q_objects, models.Q()) or Q_NONE
+
+
+def locked(meth):
+    @functools.wraps(meth)
+    def _locked_meth(self, *args, **kwargs):
+        if getattr(self, "_locked", False):
+            return meth(self, *args, **kwargs)
+
+        with transaction.atomic():
+            type(self).objects.filter(id=self.id).select_for_update().only("id").get()
+            self.refresh_from_db()
+
+            # Mark as locked in case we are called recursively
+            self._locked = True
+            try:
+                return meth(self, *args, **kwargs)
+            finally:
+                self._locked = False
+
+    return _locked_meth
+
+
+def only_if_state(state, no_op_states=frozenset()):
+    def only_if_state_decorator(meth):
+        def only_if_state_wrapper(self, *args, no_op_if_already_in_state=False, **kwargs):
+            if self.state in no_op_states:
+                return
+            states = (state,) if isinstance(state, str) else tuple(state)
+            if self.state not in states:
+                raise RuntimeError(
+                    "%s cannot execute %s as in state %s not one of %s"
+                    % (self, meth.__name__, self.state, states)
+                )
+            return meth(self, *args, **kwargs)
+
+        return only_if_state_wrapper
+
+    return only_if_state_decorator

bgtask/views.py

@@ -1,6 +1,5 @@
 from django.core.exceptions import ValidationError
 from django.db.models import Q
-from django.forms.models import model_to_dict
 from django.http import Http404, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render
 
@@ -31,18 +30,12 @@ def background_tasks_view(request):
     tasks = request.GET.get("tasks", "")
     object_id = request.GET.get("object_id", None)
     if tasks is None and object_id is None:
-        return HttpResponseBadRequest(
-            "Must pass 'tasks' or 'object_id' as a query parameter"
-        )
+        return HttpResponseBadRequest("Must pass 'tasks' or 'object_id' as a query parameter")
+    task_ids = tasks.split(",")
+    task_ids_q = Q(id__in=task_ids) if tasks else Q_NONE
+    object_id_q = Q(acted_on_object_id=object_id) if object_id is not None else Q_NONE
+    tasks = BackgroundTask.objects.filter(task_ids_q | object_id_q).order_by("-created")
     try:
-        task_ids = tasks.split(",")
-        task_ids_q = Q(id__in=task_ids) if tasks else Q_NONE
-        object_id_q = (
-            Q(acted_on_object_id=object_id) if object_id is not None else Q_NONE
-        )
-        tasks = BackgroundTask.objects.filter(task_ids_q | object_id_q).order_by(
-            "-created"
-        )
         if len(tasks) == 0:
             raise ValidationError("Unfound tasks")
     except ValidationError:
@@ -52,6 +45,8 @@ def background_tasks_view(request):
 
     accepts = request.headers.get("Accept", "").split(",")
 
+    tasks.add_position_in_queue()
+
     if "application/json" in accepts:
         return background_tasks_view_json(tasks)
 

{django_admin_background_task-0.4.0.dist-info → django_admin_background_task-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: django-admin-background-task
-Version: 0.4.0
+Version: 0.4.3
 Summary: A set of tools for django apps to persist and monitor the status of background tasks
 Home-page: https://github.com/daphtdazz/django-bgtask
 License: Apache-2.0
@@ -58,4 +58,5 @@ urlpatterns = [
 from bgtask.models import BackgroundTask
 
 BackgroundTask.new(name)
+```
 

{django_admin_background_task-0.4.0.dist-info → django_admin_background_task-0.4.3.dist-info}/RECORD

@@ -8,11 +8,11 @@ bgtask/migrations/0001_initial.py,sha256=TPuplGYJtqHJ7h4-A-nINMmvRD9HYvfhOY4bZyi
 bgtask/migrations/0002_backgroundtask_namespace_alter_backgroundtask_name.py,sha256=XIG_gIvdA73tz0NRbNkGU_ieKde3ZtQltPeLdslD5Xo,968
 bgtask/migrations/0003_backgroundtask_queued_at.py,sha256=Hp0WeQLKP0R5jNeEAgVlsx-7jgIzbnIkqkPUGsenK8o,439
 bgtask/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bgtask/model_admin.py,sha256=loEh9uPUZbC19_daPoPfHNuLrI42oAA930J5IEH_1-w,5057
-bgtask/models.py,sha256=meuAEcYP--YocGmQ8kVcvK7zCoom9vGSQX0EWiuSn3U,10566
+bgtask/model_admin.py,sha256=Of0Le1IV3482HurWyi-FB2TquuRXtDxDK0tEtc-OFtE,5090
+bgtask/models.py,sha256=FOpUaLu72OXRTDyNSe1gKOp5okWMFboDKTlXkR1Cyn0,11616
 bgtask/static/bgtask/js/bgtask-once.js,sha256=DBerpxPLP21IH--HpBxVFk3kN52FtBiuyJsDCjg7mCA,746
 bgtask/static/bgtask/js/bgtask.js,sha256=6fVepM6AY_a3-I5ZABfaCaf7_g8IkClOFq9ZxG9GsX4,15366
-bgtask/templates/bgtask/admin/change_list.html,sha256=pxajrUXZvsBfSM4Lh3HLjdjGNNO7lJPaTkUwZ_zbWEc,1080
+bgtask/templates/bgtask/admin/change_list.html,sha256=cb5v5jkTsHgU7cZRQchxra0aXUXAvlBlxPZ0ZjLQPNc,1143
 bgtask/templates/bgtask/bg_changelist_status_column.html,sha256=SA7EAL50oBrcRPGstXROXkFJ541vYA8RlHidM3srTPU,844
 bgtask/templates/bgtask/bg_completed_column.html,sha256=Q4l3EQaOXHYVOzv0jpx3ww3Wa2Bb33RyZp3RSWI1zog,788
 bgtask/templates/bgtask/bgtask_templates.html,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -20,10 +20,11 @@ bgtask/templates/bgtask/bgtask_view.html,sha256=AeubqERFF14PHWjInBD39wGImnRx4taS
 bgtask/templates/bgtask/progress.html,sha256=VVYbrunGasOshRlz8Nm8hFPxqcVtrpaW1e2QxzAMcMM,166
 bgtask/templatetags/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bgtask/templatetags/bgtask.py,sha256=WUEWlQX5AarbXJBV6EGp-khT1zegVwGNigSKRF5mCTE,413
-bgtask/tests/test_bgtask.py,sha256=9AsBppPphaIdphuB-pHuQjgLhYNp8ySWtmcWF2uQl3Q,1090
+bgtask/tests/test_bgtask.py,sha256=pepgFBvRWNqmOdOFPkp4MC_ctWmYPYGpisb94OUUA9I,1882
 bgtask/urls.py,sha256=DXXjgj18LTSCq2-xZf23-pkgO4RmZglv67mgrp-fDyk,161
-bgtask/views.py,sha256=tPblidBq8mIv9cbyAfzJfNlM-I-Z11XVoobdA3DZBVM,1765
-django_admin_background_task-0.4.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-django_admin_background_task-0.4.0.dist-info/METADATA,sha256=8IjDh7pOq9dKgYCqPr_fbHgI4rZNmRDeUB2NApdmCT4,1375
-django_admin_background_task-0.4.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-django_admin_background_task-0.4.0.dist-info/RECORD,,
+bgtask/utils.py,sha256=lc3V276dEper-3d5xgwTdsfmsKOLv9KShrPkFCkPzlg,1789
+bgtask/views.py,sha256=O-8J8VBLXnbBkbDseu-5y0hGt3aU5r-awsTq8Rdg7mc,1670
+django_admin_background_task-0.4.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+django_admin_background_task-0.4.3.dist-info/METADATA,sha256=T709ili5frQROuOnoU16OnODFaS9LEpJskRlCN0doQo,1379
+django_admin_background_task-0.4.3.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+django_admin_background_task-0.4.3.dist-info/RECORD,,