pydocket 0.3.2__tar.gz → 0.4.0__tar.gz

{pydocket-0.3.2 → pydocket-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.3.2
+Version: 0.4.0
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

{pydocket-0.3.2 → pydocket-0.4.0}/src/docket/docket.py

@@ -132,6 +132,9 @@ class Docket:
                 - "redis://user:password@localhost:6379/0?ssl=true"
                 - "rediss://localhost:6379/0"
                 - "unix:///path/to/redis.sock"
+            heartbeat_interval: How often workers send heartbeat messages to the docket.
+            missed_heartbeats: How many heartbeats a worker can miss before it is
+                considered dead.
         """
         self.name = name
         self.url = url
@@ -305,6 +308,9 @@ class Docket:
     def stream_key(self) -> str:
         return f"{self.name}:stream"
 
+    def known_task_key(self, key: str) -> str:
+        return f"{self.name}:known:{key}"
+
     def parked_task_key(self, key: str) -> str:
         return f"{self.name}:{key}"
 
@@ -340,30 +346,36 @@ class Docket:
             when = execution.when
 
             async with self.redis() as redis:
-                # if the task is already in the queue, retain it
-                if await redis.zscore(self.queue_key, key) is not None:
+                # if the task is already in the queue or stream, retain it
+                if await redis.exists(self.known_task_key(key)):
+                    logger.debug(
+                        "Task %r is already in the queue or stream, skipping schedule",
+                        key,
+                        extra=self.labels(),
+                    )
                     return
 
-                if when <= datetime.now(timezone.utc):
-                    await redis.xadd(self.stream_key, message)  # type: ignore[arg-type]
-                else:
-                    async with redis.pipeline() as pipe:
+                async with redis.pipeline() as pipe:
+                    pipe.set(self.known_task_key(key), when.timestamp())
+
+                    if when <= datetime.now(timezone.utc):
+                        pipe.xadd(self.stream_key, message)  # type: ignore[arg-type]
+                    else:
                         pipe.hset(self.parked_task_key(key), mapping=message)  # type: ignore[arg-type]
                         pipe.zadd(self.queue_key, {key: when.timestamp()})
-                        await pipe.execute()
+
+                    await pipe.execute()
 
         TASKS_SCHEDULED.add(1, {**self.labels(), **execution.general_labels()})
 
     async def cancel(self, key: str) -> None:
         with tracer.start_as_current_span(
             "docket.cancel",
-            attributes={
-                **self.labels(),
-                "docket.key": key,
-            },
+            attributes={**self.labels(), "docket.key": key},
         ):
             async with self.redis() as redis:
                 async with redis.pipeline() as pipe:
+                    pipe.delete(self.known_task_key(key))
                     pipe.delete(self.parked_task_key(key))
                     pipe.zrem(self.queue_key, key)
                     await pipe.execute()

{pydocket-0.3.2 → pydocket-0.4.0}/src/docket/worker.py

@@ -377,6 +377,9 @@ class Worker:
 
         execution = Execution.from_message(function, message)
 
+        async with self.docket.redis() as redis:
+            await redis.delete(self.docket.known_task_key(execution.key))
+
         log_context = {**log_context, **execution.specific_labels()}
         counter_labels = {**self.labels(), **execution.general_labels()}
 

{pydocket-0.3.2 → pydocket-0.4.0}/tests/test_fundamentals.py

@@ -79,7 +79,7 @@ async def test_scheduled_execution(
     assert when <= now()
 
 
-async def test_adding_is_itempotent(
+async def test_adding_is_idempotent(
     docket: Docket, worker: Worker, the_task: AsyncMock, now: Callable[[], datetime]
 ):
     """docket should allow for rescheduling a task for later"""
@@ -159,6 +159,77 @@ async def test_rescheduling_by_name(
     assert later <= now()
 
 
+async def test_task_keys_are_idempotent_in_the_future(
+    docket: Docket, worker: Worker, the_task: AsyncMock, now: Callable[[], datetime]
+):
+    """docket should only allow one task with the same key to be scheduled or due"""
+
+    key = f"my-cool-task:{uuid4()}"
+
+    soon = now() + timedelta(milliseconds=10)
+    await docket.add(the_task, when=soon, key=key)("a", "b", c="c")
+    await docket.add(the_task, when=now(), key=key)("d", "e", c="f")
+
+    await worker.run_until_finished()
+
+    the_task.assert_awaited_once_with("a", "b", c="c")
+    the_task.reset_mock()
+
+    # It should be fine to run it afterward
+    await docket.add(the_task, key=key)("d", "e", c="f")
+
+    await worker.run_until_finished()
+
+    the_task.assert_awaited_once_with("d", "e", c="f")
+
+
+async def test_task_keys_are_idempotent_between_the_future_and_present(
+    docket: Docket, worker: Worker, the_task: AsyncMock, now: Callable[[], datetime]
+):
+    """docket should only allow one task with the same key to be scheduled or due"""
+
+    key = f"my-cool-task:{uuid4()}"
+
+    soon = now() + timedelta(milliseconds=10)
+    await docket.add(the_task, when=now(), key=key)("a", "b", c="c")
+    await docket.add(the_task, when=soon, key=key)("d", "e", c="f")
+
+    await worker.run_until_finished()
+
+    the_task.assert_awaited_once_with("a", "b", c="c")
+    the_task.reset_mock()
+
+    # It should be fine to run it afterward
+    await docket.add(the_task, key=key)("d", "e", c="f")
+
+    await worker.run_until_finished()
+
+    the_task.assert_awaited_once_with("d", "e", c="f")
+
+
+async def test_task_keys_are_idempotent_in_the_present(
+    docket: Docket, worker: Worker, the_task: AsyncMock, now: Callable[[], datetime]
+):
+    """docket should only allow one task with the same key to be scheduled or due"""
+
+    key = f"my-cool-task:{uuid4()}"
+
+    await docket.add(the_task, when=now(), key=key)("a", "b", c="c")
+    await docket.add(the_task, when=now(), key=key)("d", "e", c="f")
+
+    await worker.run_until_finished()
+
+    the_task.assert_awaited_once_with("a", "b", c="c")
+    the_task.reset_mock()
+
+    # It should be fine to run it afterward
+    await docket.add(the_task, key=key)("d", "e", c="f")
+
+    await worker.run_until_finished()
+
+    the_task.assert_awaited_once_with("d", "e", c="f")
+
+
 async def test_cancelling_future_task(
     docket: Docket, worker: Worker, the_task: AsyncMock, now: Callable[[], datetime]
 ):
@@ -696,7 +767,8 @@ async def test_striking_entire_parameters(
             call(customer_id="123", order_id="456"),
             call(customer_id="456", order_id="789"),
             # customer_id == 789 is stricken
-        ]
+        ],
+        any_order=True,
     )
     the_task.reset_mock()
 
@@ -705,7 +777,8 @@ async def test_striking_entire_parameters(
         [
             call(customer_id="456", order_id="012"),
             # customer_id == 789 is stricken
-        ]
+        ],
+        any_order=True,
     )
     another_task.reset_mock()
 
@@ -725,7 +798,8 @@ async def test_striking_entire_parameters(
             # customer_id == 123 is stricken
             call(customer_id="456", order_id="789"),
             # customer_id == 789 is stricken
-        ]
+        ],
+        any_order=True,
     )
     the_task.reset_mock()
 
@@ -734,7 +808,8 @@ async def test_striking_entire_parameters(
         [
             call(customer_id="456", order_id="012"),
             # customer_id == 789 is stricken
-        ]
+        ],
+        any_order=True,
     )
     another_task.reset_mock()
 
@@ -754,7 +829,8 @@ async def test_striking_entire_parameters(
             call(customer_id="123", order_id="456"),
             call(customer_id="456", order_id="789"),
             # customer_id == 789 is still stricken
-        ]
+        ],
+        any_order=True,
     )
 
     assert another_task.call_count == 1
@@ -762,7 +838,8 @@ async def test_striking_entire_parameters(
         [
             call(customer_id="456", order_id="012"),
             # customer_id == 789 is still stricken
-        ]
+        ],
+        any_order=True,
     )
 
 
@@ -787,7 +864,8 @@ async def test_striking_tasks_for_specific_parameters(
             # b <= 2 is stricken, so b=1 is out
             # b <= 2 is stricken, so b=2 is out
             call("a", b=3),
-        ]
+        ],
+        any_order=True,
     )
     the_task.reset_mock()
 
@@ -797,7 +875,8 @@ async def test_striking_tasks_for_specific_parameters(
             call("d", b=1),
             call("d", b=2),
             call("d", b=3),
-        ]
+        ],
+        any_order=True,
     )
     another_task.reset_mock()
 
@@ -818,7 +897,8 @@ async def test_striking_tasks_for_specific_parameters(
             call("a", b=1),
             call("a", b=2),
             call("a", b=3),
-        ]
+        ],
+        any_order=True,
     )
 
     assert another_task.call_count == 3
@@ -827,7 +907,8 @@ async def test_striking_tasks_for_specific_parameters(
             call("d", b=1),
             call("d", b=2),
             call("d", b=3),
-        ]
+        ],
+        any_order=True,
     )
 
 

{pydocket-0.3.2 → pydocket-0.4.0}/tests/test_worker.py

@@ -388,7 +388,7 @@ async def test_perpetual_tasks_are_scheduled_close_to_target_time(
     average = total / len(intervals)
 
     # even with a variable duration, Docket attempts to schedule them equally
-    assert timedelta(milliseconds=45) <= average <= timedelta(milliseconds=70)
+    assert timedelta(milliseconds=45) <= average <= timedelta(milliseconds=75)
 
 
 async def test_worker_can_exit_from_perpetual_tasks_that_queue_further_tasks(