pydocket 0.26.2__tar.gz → 0.27.0__tar.gz

{pydocket-0.26.2 → pydocket-0.27.0}/PKG-INFO

@@ -1,24 +1,15 @@
 Metadata-Version: 2.5
 Name: pydocket
-Version: 0.26.2
+Version: 0.27.0
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://docket.lol/
 Project-URL: Documentation, https://docket.lol/en/latest/
 Project-URL: Repository, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues
 Author-email: Chris Guidry <guid@omg.lol>
-License: # Released under MIT License
-        
-        Copyright (c) 2025 Chris Guidry.
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+License-Expression: MIT
 License-File: LICENSE
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{pydocket-0.26.2 → pydocket-0.27.0}/pyproject.toml

@@ -8,7 +8,8 @@ dynamic = ["version"]
 description = "A distributed background task system for Python functions"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"
-license = { file = "LICENSE" }
+license = "MIT"
+license-files = ["LICENSE"]
 authors = [{ name = "Chris Guidry", email = "guid@omg.lol" }]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -18,7 +19,6 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.14",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Typing :: Typed",
 ]

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/_execution_scripts.py

@@ -94,4 +94,5 @@ async def _cancel_task(
     task_key: Arg[str],
     completed_at: Arg[str],
     state_payload: Arg[str],
-) -> bytes: ...
+    expected_generation: Arg[int],
+) -> bytes | str: ...

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/cli/__init__.py

@@ -225,6 +225,18 @@ def worker(
             envvar="DOCKET_FALLBACK_TASK",
         ),
     ] = None,
+    dependencies: Annotated[
+        str | None,
+        typer.Option(
+            "--dependencies",
+            help=(
+                "Path to a worker dependency collection (module:member). "
+                "The collection is a mapping or sequence of Dependency instances "
+                "(e.g., Depends(fn)) that run around every task."
+            ),
+            envvar="DOCKET_WORKER_DEPENDENCIES",
+        ),
+    ] = None,
 ) -> None:
     asyncio.run(
         Worker.run(
@@ -244,6 +256,7 @@ def worker(
             metrics_port=metrics_port,
             tasks=tasks,
             fallback_task=fallback_task,
+            dependencies=dependencies,
         )
     )
 

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/dependencies/_perpetual.py

@@ -20,7 +20,7 @@ if TYPE_CHECKING:  # pragma: no cover
     from ..docket import Docket
     from ..execution import Execution
 
-from ..execution import Disposition
+from ..execution import Disposition, ExecutionState
 from ..instrumentation import TASKS_PERPETUATED, TASKS_SUPERSEDED
 
 logger = logging.getLogger("docket.dependencies")
@@ -114,13 +114,20 @@ class Perpetual(CompletionHandler["Perpetual"]):
 
     async def on_complete(self, execution: Execution, outcome: TaskOutcome) -> bool:
         """Handle completion by scheduling the next execution."""
+        docket = current_docket.get()
+
         if self.cancelled:
-            docket = current_docket.get()
+            # Cancel under this attempt's generation, so a replace made while
+            # it ran survives the stop.
             async with docket.redis() as redis:
-                await docket._cancel(redis, execution.key)
+                cancelled = await docket._cancel(
+                    redis, execution.key, execution.generation
+                )
+            if not cancelled:
+                await self._end_superseded(execution, outcome)
+                return True
             return False
 
-        docket = current_docket.get()
         worker = current_worker.get()
 
         if self._next_when:
@@ -140,19 +147,7 @@ class Perpetual(CompletionHandler["Perpetual"]):
         )
 
         if successor.disposition is Disposition.SUPERSEDED:
-            TASKS_SUPERSEDED.add(
-                1,
-                {
-                    **worker.labels(),
-                    **execution.general_labels(),
-                    "docket.where": "on_complete",
-                },
-            )
-            logger.info(
-                "↬ [%s] %s (superseded)",
-                format_duration(outcome.duration.total_seconds()),
-                execution.call_repr(),
-            )
+            await self._end_superseded(execution, outcome)
             return True
 
         TASKS_PERPETUATED.add(1, {**worker.labels(), **execution.general_labels()})
@@ -172,3 +167,31 @@ class Perpetual(CompletionHandler["Perpetual"]):
         )
 
         return True
+
+    async def _end_superseded(self, execution: Execution, outcome: TaskOutcome) -> None:
+        """Ends a run whose stop or reschedule gave way to a replace.  It
+        publishes no state, so a caller waiting on the key waits for the
+        replacement instead of taking this run as its end."""
+        worker = current_worker.get()
+        TASKS_SUPERSEDED.add(
+            1,
+            {
+                **worker.labels(),
+                **execution.general_labels(),
+                "docket.where": "on_complete",
+            },
+        )
+        logger.info(
+            "↬ [%s] %s (superseded)",
+            format_duration(outcome.duration.total_seconds()),
+            execution.call_repr(),
+        )
+        exception = outcome.exception
+        if exception is None:
+            await execution._mark_as_terminal(ExecutionState.COMPLETED, publish=False)
+        else:
+            await execution._mark_as_terminal(
+                ExecutionState.FAILED,
+                error=f"{type(exception).__name__}: {exception}",
+                publish=False,
+            )

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/dependencies/_retry.py

@@ -17,7 +17,8 @@ from ._base import (
 if TYPE_CHECKING:  # pragma: no cover
     from ..execution import Execution
 
-from ..instrumentation import TASKS_RETRIED
+from ..execution import Disposition, ExecutionState
+from ..instrumentation import TASKS_RETRIED, TASKS_SUPERSEDED
 
 logger = logging.getLogger("docket.dependencies")
 
@@ -105,12 +106,16 @@ class Retry(FailureHandler["Retry"]):
         # Pass both replace=True and the original message_id.  The reschedule
         # branch handles the ACK+XDEL of the failed message when message_id is
         # set; replace=True keeps correct behavior in the (otherwise
-        # impossible) case where message_id is None.
-        await execution.schedule(replace=True, reschedule_message=execution.message_id)
+        # impossible) case where message_id is None.  The retry carries this
+        # attempt's generation, so a replace made while it ran keeps its own
+        # time and arguments instead of giving way to the retry.
+        disposition = await execution.schedule(
+            replace=True,
+            reschedule_message=execution.message_id,
+            expected_generation=execution.generation,
+        )
 
         worker = current_worker.get()
-        TASKS_RETRIED.add(1, {**worker.labels(), **execution.general_labels()})
-
         logger.error(
             "↩ [%s] %s",
             format_duration(outcome.duration.total_seconds()),
@@ -118,6 +123,34 @@ class Retry(FailureHandler["Retry"]):
             exc_info=outcome.exception,
         )
 
+        if disposition is Disposition.SUPERSEDED:
+            TASKS_SUPERSEDED.add(
+                1,
+                {
+                    **worker.labels(),
+                    **execution.general_labels(),
+                    "docket.where": "retry",
+                },
+            )
+            logger.info(
+                "↬ [%s] %s (superseded)",
+                format_duration(outcome.duration.total_seconds()),
+                execution.call_repr(),
+            )
+            # The schedule left the failed delivery pending; ending it as
+            # failed acknowledges it without touching the newer run's state.
+            # It publishes no state, so a caller waiting on the key waits for
+            # the replacement instead of taking this failure as its end.
+            exception = outcome.exception
+            await execution._mark_as_terminal(
+                ExecutionState.FAILED,
+                error=f"{type(exception).__name__}: {exception}",
+                publish=False,
+            )
+            return True
+
+        TASKS_RETRIED.add(1, {**worker.labels(), **execution.general_labels()})
+
         logger.info(
             "↫ [%s] %s",
             format_duration(outcome.duration.total_seconds()),

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/docket.py

@@ -27,6 +27,7 @@ from ._docket_snapshot import DocketSnapshotMixin
 from ._docket_snapshot import RunningExecution as RunningExecution
 from ._docket_snapshot import WorkerInfo as WorkerInfo
 from ._execution_scripts import _cancel_task
+from ._lua import Arg, Key, redis_script
 from ._redis import (
     PubSubClient,
     RedisClient,
@@ -68,6 +69,24 @@ logger: logging.Logger = logging.getLogger(__name__)
 tracer: trace.Tracer = trace.get_tracer(__name__)
 
 
+@redis_script
+async def _clear(
+    redis: RedisClient,
+    *,
+    stream_key: Key[str],
+    queue_key: Key[str],
+    docket_prefix: Arg[str],
+    completed_at: Arg[str],
+    ttl_seconds: Arg[int],
+    batch: Arg[int],
+) -> int: ...
+
+
+# How many queue entries and stream messages one clear.lua call removes, so
+# that no one call blocks Redis for long.
+CLEAR_BATCH = 1000
+
+
 P = ParamSpec("P")
 R = TypeVar("R")
 
@@ -912,7 +931,9 @@ class Docket(DocketSnapshotMixin):
             if "BUSYGROUP" not in str(e):
                 raise  # pragma: no cover
 
-    async def _cancel(self, redis: RedisClient, key: str) -> None:
+    async def _cancel(
+        self, redis: RedisClient, key: str, expected_generation: int = 0
+    ) -> bool:
         """Cancel a task atomically.
 
         Handles cancellation regardless of task location:
@@ -925,6 +946,17 @@ class Docket(DocketSnapshotMixin):
         waiter streams) clean up via the cancel pub/sub channel
         published by ``Docket.cancel`` -- Docket itself stays unaware of any
         dependency-specific storage.
+
+        Args:
+            redis: The connection to run the cancel on.
+            key: The key of the task to cancel.
+            expected_generation: The generation the caller believes holds
+                ``key``.  When it is non-zero and Redis holds a newer one,
+                the cancel leaves the key alone.  0 skips the check.
+
+        Returns:
+            False when a newer generation holds the key and nothing changed,
+            otherwise True.
         """
         # Create tombstone with CANCELLED state
         completed_at = datetime.now(timezone.utc).isoformat()
@@ -938,8 +970,7 @@ class Docket(DocketSnapshotMixin):
             }
         )
 
-        # Execute the cancellation script
-        await _cancel_task(
+        reply = await _cancel_task(
             redis,
             stream_key=self.stream_key,
             known_key=self.known_task_key(key),
@@ -952,7 +983,12 @@ class Docket(DocketSnapshotMixin):
             task_key=key,
             completed_at=completed_at,
             state_payload=state_payload,
+            expected_generation=expected_generation,
         )
+        # The runs hash belongs to the newer generation, so it keeps its own
+        # lifetime.
+        if reply in (b"SUPERSEDED", "SUPERSEDED"):
+            return False
 
         # Apply TTL or delete tombstone based on execution_ttl
         if self.execution_ttl:
@@ -961,6 +997,7 @@ class Docket(DocketSnapshotMixin):
         else:
             # execution_ttl=0 means no observability - delete tombstone immediately
             await redis.delete(task_runs_key)
+        return True
 
     async def strike(
         self,
@@ -1024,8 +1061,17 @@ class Docket(DocketSnapshotMixin):
         """Clear all queued and scheduled tasks from the docket.
 
         This removes all tasks from the stream (immediate tasks) and queue
-        (scheduled tasks), along with their associated parked data. Running
-        tasks are not affected.
+        (scheduled tasks), along with their associated parked data, and
+        cancels each task that has not started, as ``cancel()`` does: its
+        key is free for a new task at once, and a caller waiting on its
+        result receives ``ExecutionCancelled``.  Running tasks are not
+        affected, and neither are tasks that wait for a ``ConcurrencyLimit``
+        slot, which are in neither the stream nor the queue.
+
+        It clears the docket in batches, so other clients can use Redis
+        between them.  A worker can start a task between two batches, and
+        that task then runs.  A task added during a clear may or may not be
+        cleared.
 
         Returns:
             The total number of tasks that were cleared.
@@ -1034,60 +1080,21 @@ class Docket(DocketSnapshotMixin):
             "docket.clear",
             attributes=self.labels(),
         ):
+            total_cleared = 0
             async with self.redis() as redis:
-                async with redis.pipeline() as pipeline:
-                    # Get counts before clearing
-                    pipeline.xlen(self.stream_key)
-                    pipeline.zcard(self.queue_key)
-                    pipeline.zrange(self.queue_key, 0, -1)
-
-                    stream_count: int
-                    queue_count: int
-                    scheduled_keys: list[bytes]
-                    stream_count, queue_count, scheduled_keys = await pipeline.execute()
-
-                # Get keys from stream messages before trimming
-                stream_keys: list[str] = []
-                if stream_count > 0:
-                    # Read all messages from the stream
-                    messages = await redis.xrange(self.stream_key, "-", "+")
-                    for message_id, fields in messages:
-                        # Extract the key field from the message
-                        if b"key" in fields:  # pragma: no branch
-                            stream_keys.append(fields[b"key"].decode())
-
-                async with redis.pipeline() as pipeline:
-                    # Clear all data
-                    # Trim stream to 0 messages instead of deleting it to preserve consumer group
-                    if stream_count > 0:
-                        pipeline.xtrim(self.stream_key, maxlen=0, approximate=False)
-                    pipeline.delete(self.queue_key)
-
-                    # Clear parked task data and known task keys for scheduled tasks
-                    for key_bytes in scheduled_keys:
-                        task_key = key_bytes.decode()
-                        pipeline.delete(self.parked_task_key(task_key))
-                        pipeline.delete(self.known_task_key(task_key))
-                        pipeline.delete(self.stream_id_key(task_key))
-
-                        # Handle runs hash: set TTL or delete based on execution_ttl
-                        task_runs_key = self.runs_key(task_key)
-                        if self.execution_ttl:
-                            ttl_seconds = int(self.execution_ttl.total_seconds())
-                            pipeline.expire(task_runs_key, ttl_seconds)
-                        else:
-                            pipeline.delete(task_runs_key)
-
-                    # Handle runs hash for immediate tasks from stream
-                    for task_key in stream_keys:
-                        task_runs_key = self.runs_key(task_key)
-                        if self.execution_ttl:
-                            ttl_seconds = int(self.execution_ttl.total_seconds())
-                            pipeline.expire(task_runs_key, ttl_seconds)
-                        else:
-                            pipeline.delete(task_runs_key)
-
-                    await pipeline.execute()
-
-                    total_cleared = stream_count + queue_count
-                    return total_cleared
+                # A call that clears less than one batch emptied the stream
+                # and the queue, so the clear ends there even while new tasks
+                # keep arriving.
+                cleared = CLEAR_BATCH
+                while cleared >= CLEAR_BATCH:
+                    cleared = await _clear(
+                        redis,
+                        stream_key=self.stream_key,
+                        queue_key=self.queue_key,
+                        docket_prefix=self.prefix,
+                        completed_at=datetime.now(timezone.utc).isoformat(),
+                        ttl_seconds=int(self.execution_ttl.total_seconds()),
+                        batch=CLEAR_BATCH,
+                    )
+                    total_cleared += cleared
+            return total_cleared

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/execution.py

@@ -205,8 +205,8 @@ class Disposition(enum.Enum):
     SUPERSEDED = "superseded"
     """A newer schedule already holds this key, so the attempt left it alone.
     Only possible when the caller states the generation it expects, which
-    ``Perpetual`` does when it reschedules the attempt that just finished;
-    ``Docket.add`` and ``Docket.replace`` never produce it."""
+    ``Perpetual`` and ``Retry`` do when they reschedule the attempt that just
+    finished; ``Docket.add`` and ``Docket.replace`` never produce it."""
 
     FAILED = "failed"
     """The Redis command scheduling this task returned an error.  Only
@@ -279,6 +279,11 @@ class Execution:
         # False, and the worker can ack defensively.
         self._acked: bool = False
 
+        # True once ``_mark_as_terminal`` has ended this run.  A completion
+        # handler that ends the run itself, as a superseded Perpetual does,
+        # sets it, and the worker then leaves out its own terminal write.
+        self._ended: bool = False
+
         # True when the last claim() refused this task as cancelled.  The worker
         # reads it because a superseded task's state is also CANCELLED when the
         # newer generation was cancelled.
@@ -681,6 +686,7 @@ class Execution:
         *,
         error: str | None = None,
         result_key: str | None = None,
+        publish: bool = True,
     ) -> None:
         """Mark task as having reached a terminal state.
 
@@ -688,6 +694,9 @@ class Execution:
             state: The terminal state (COMPLETED, FAILED, or CANCELLED)
             error: Optional error message (for FAILED state)
             result_key: Optional key where the result/exception is stored
+            publish: Whether to publish the terminal-state event.  A run that
+                a replace superseded passes False, because the key's task has
+                not ended.
 
         Uses a Lua script to atomically check supersession, write the
         terminal state, publish the completion event, delete the progress
@@ -722,7 +731,7 @@ class Execution:
         }
         if error:
             state_payload_data["error"] = error
-        state_payload = json.dumps(state_payload_data)
+        state_payload = json.dumps(state_payload_data) if publish else ""
 
         # Set ``_acked = True`` *before* the awaited Lua call: the server
         # commit is the source of truth, so once we hand the call off we own
@@ -731,6 +740,7 @@ class Execution:
         # and let the worker safety net overwrite the committed terminal
         # state with FAILED/None.
         self._acked = True
+        self._ended = True
 
         with self._maybe_suppress_instrumentation():
             async with self.docket.redis() as redis:

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/lua/cancel_task.lua

@@ -9,10 +9,22 @@ local state_channel = KEYS[8]
 local task_key = ARGV[1]
 local completed_at = ARGV[2]
 local state_payload = ARGV[3]
+local expected_generation = tonumber(ARGV[4])
 
 -- TODO: Remove known_key / parked_key / stream_id_key handling in
 -- v0.14.0 (legacy key locations).
 
+-- A run that cancels its own key (a Perpetual that stops itself) passes
+-- the generation it holds.  A newer stored generation means a replace took
+-- the key while that run ran, and the cancel must leave the replacement
+-- alone.  0 means "no check", for a cancel from outside any run.
+if expected_generation > 0 then
+    local stored = redis.call('HGET', runs_key, 'generation')
+    if stored and tonumber(stored) > expected_generation then
+        return 'SUPERSEDED'
+    end
+end
+
 -- Get stream ID (check new location first, then legacy)
 local message_id = redis.call('HGET', runs_key, 'stream_id')
 

pydocket-0.27.0/src/docket/lua/clear.lua

@@ -0,0 +1,95 @@
+local stream_key = KEYS[1]
+local queue_key = KEYS[2]
+local docket_prefix = ARGV[1]
+local completed_at = ARGV[2]
+local ttl_seconds = tonumber(ARGV[3])
+local batch = tonumber(ARGV[4])
+
+-- Clearing a docket cancels every task that has not started, the way
+-- cancel_task.lua cancels one, so that each cleared key is free for a new
+-- add() at once and a caller waiting on its result receives the cancelled
+-- state.  Each call clears at most `batch` queue entries and `batch` stream
+-- messages, and the caller calls again until one clears less than that.  A
+-- script that cleared a large backlog at once would block every other
+-- client for as long as it ran.  Each call reads its tasks and removes them, so a task
+-- added in between cannot lose its message and keep its key reserved.  A
+-- worker that read a queued task's message before the clear, but has not
+-- claimed it, finds the task cancelled, and claim.lua refuses it.
+--
+-- The stream also holds the messages of tasks that are running.  Their
+-- messages go too, but their runs hashes stay as they are, because each
+-- one's worker records how it ends.
+
+-- Task keys are user-supplied, and JSON allows no control character in a
+-- string, so this escapes every one of them.  cjson is not available on the
+-- in-memory backend.
+local function json_escape(s)
+    s = s:gsub('[\\"]', '\\%0')
+    s = s:gsub('%c', function(c)
+        return string.format('\\u%04x', c:byte())
+    end)
+    return s
+end
+
+local task_keys = {}
+local seen = {}
+local function remember(task_key)
+    if not seen[task_key] then
+        seen[task_key] = true
+        task_keys[#task_keys + 1] = task_key
+    end
+end
+
+-- Only a scheduled task has parked data, at the docket's prefix and its
+-- key.  A key from the stream gets no DEL there: for an immediate task with
+-- a key like "stream", that DEL would remove the stream itself.
+local scheduled = {}
+local queued = redis.call('ZRANGE', queue_key, 0, batch - 1)
+for _, task_key in ipairs(queued) do
+    scheduled[task_key] = true
+    remember(task_key)
+end
+if #queued > 0 then
+    redis.call('ZREM', queue_key, unpack(queued))
+end
+
+-- XDEL each message, not DEL the stream, so the stream keeps its consumer
+-- group.  The in-memory backend does not run XTRIM inside Lua.
+local messages = redis.call('XRANGE', stream_key, '-', '+', 'COUNT', batch)
+for _, entry in ipairs(messages) do
+    local fields = entry[2]
+    for i = 1, #fields, 2 do
+        if fields[i] == 'key' then
+            remember(fields[i + 1])
+            break
+        end
+    end
+    redis.call('XDEL', stream_key, entry[1])
+end
+
+local prefix = docket_prefix .. ':'
+for _, task_key in ipairs(task_keys) do
+    if scheduled[task_key] then
+        redis.call('DEL', prefix .. task_key)
+    end
+    -- TODO: Remove in v0.14.0 (legacy key locations).
+    redis.call('DEL', prefix .. 'known:' .. task_key, prefix .. 'stream-id:' .. task_key)
+
+    local runs_key = prefix .. 'runs:' .. task_key
+    local state = redis.call('HGET', runs_key, 'state')
+    if state == 'scheduled' or state == 'queued' then
+        redis.call('HDEL', runs_key, 'known', 'stream_id')
+        redis.call('HSET', runs_key, 'state', 'cancelled', 'completed_at', completed_at)
+        redis.call('DEL', prefix .. 'progress:' .. task_key)
+        redis.call('PUBLISH', prefix .. 'state:' .. task_key,
+            '{"type":"state","key":"' .. json_escape(task_key) ..
+            '","state":"cancelled","completed_at":"' .. completed_at .. '"}')
+        if ttl_seconds > 0 then
+            redis.call('EXPIRE', runs_key, ttl_seconds)
+        else
+            redis.call('DEL', runs_key)
+        end
+    end
+end
+
+return #messages + #queued

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/lua/schedule.lua

@@ -19,19 +19,18 @@ local message_start = 9
 -- handling in v0.14.0 (legacy key locations).
 
 -- A caller that is rescheduling on behalf of the attempt it just ran
--- (Perpetual's on_complete) passes the generation it holds, so the
--- supersession check rides along with the schedule instead of costing
+-- (Perpetual's on_complete, or a Retry) passes the generation it holds, so
+-- the supersession check rides along with the schedule instead of costing
 -- its own HGET.  A newer stored generation means someone else has taken
--- the key and this schedule must not touch it.  0 means "no check", both
--- for callers that don't have a generation and for messages that predate
--- generation tracking.
+-- the key and this schedule must not touch it.  A missing runs hash does
+-- not refuse the schedule: an eviction or an older clear() can remove the
+-- hash of a run that is still going, and that run's retry or next
+-- Perpetual run went ahead before this check existed.  0 means "no check",
+-- both for callers that don't have a generation and for messages that
+-- predate generation tracking.
 if expected_generation > 0 then
     local stored = redis.call('HGET', runs_key, 'generation')
-    local stored_generation = 0
-    if stored then
-        stored_generation = tonumber(stored)
-    end
-    if stored_generation > expected_generation then
+    if stored and tonumber(stored) > expected_generation then
         return 'SUPERSEDED'
     end
 end

{pydocket-0.26.2 → pydocket-0.27.0}/src/docket/lua/terminal.lua

@@ -21,10 +21,16 @@ local extra_fields_start = 8
 -- waiting on completion don't deadlock, and we still clean up this
 -- execution's progress hash and stream entry.  We do NOT recreate or
 -- mutate the runs hash on a supersession -- the successor owns it.
+--
+-- An empty state_payload publishes nothing.  A retry that a replace
+-- superseded passes it, because the key's task has not ended, and a
+-- caller waiting on the key must wait for the replacement.
 if generation > 0 then
     local current = redis.call('HGET', runs_key, 'generation')
     if not current or tonumber(current) > generation then
-        redis.call('PUBLISH', state_channel, state_payload)
+        if state_payload ~= '' then
+            redis.call('PUBLISH', state_channel, state_payload)
+        end
         -- Only DEL the progress hash if it belongs to us (matching
         -- generation tag) or is untagged (pre-fix / pre-tracking data,
         -- preserve the prior unconditional-DEL behaviour).  A newer
@@ -56,7 +62,9 @@ else
     redis.call('DEL', runs_key)
 end
 
-redis.call('PUBLISH', state_channel, state_payload)
+if state_payload ~= '' then
+    redis.call('PUBLISH', state_channel, state_payload)
+end
 redis.call('DEL', progress_key)
 if message_id ~= '' then
     redis.call('XACK', stream_key, worker_group_name, message_id)