django-aiogram 4.0.0.dev0__py3-none-any.whl

django_aiogram/management/commands/start_tgbot.py

@@ -0,0 +1,308 @@
+"""Run the bot: receive updates, and consume the queue Django writes to.
+
+This is the long-running process a bot container is built around. It owns two
+things at once — whatever brings updates in, and the consumer that drains the
+Redis queue — and has to shut both down cleanly when the container stops.
+"""
+
+import contextlib
+import logging
+import signal
+import threading
+from argparse import ArgumentParser
+from collections.abc import Callable
+from types import FrameType
+from typing import Any
+
+from django.core.management import BaseCommand, CommandError
+
+from django_aiogram import bot
+from django_aiogram.config.enums import UpdateMode
+from django_aiogram.config.settings import SETTINGS_NAME, coerce_bool, conf
+from django_aiogram.consumer.delivery import Delivery, get_delivery
+from django_aiogram.consumer.webhook import MODES, current_mode
+from django_aiogram.eventlog.events import worker_identity
+from django_aiogram.eventlog.recorder import recorder
+from django_aiogram.redis import read_timeout
+
+logger = logging.getLogger('django_aiogram')
+
+#: what signal.signal returns: a handler, one of the SIG_* constants, or None
+Handler = Callable[[int, FrameType | None], Any] | int | None
+
+
+class Command(BaseCommand):
+    """Start the bot and the queue consumer, and stop them together."""
+
+    help = 'Start telegram bot'
+
+    #: what --idle waits on; tests replace it so they can end the wait
+    idle_event: threading.Event | None = None
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        """Declare --mode and --idle."""
+        parser.add_argument(
+            '--mode',
+            choices=sorted(MODES),
+            default=None,
+            help=(
+                "how updates reach the bot for this run. Defaults to TELEGRAM_BOT['MODE'] "
+                "(env: DJANGO_AIOGRAM_MODE), itself 'polling'. In webhook mode this "
+                'process consumes the queue and never calls getUpdates, because the updates '
+                'arrive over HTTP instead.'
+            ),
+        )
+        parser.add_argument(
+            '--idle',
+            action='store_true',
+            help=(
+                'When the bot is disabled, block instead of exiting. Useful under '
+                'restart policies that treat a clean exit as a crash loop.'
+            ),
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        """Receive updates, drain the queue, and unwind both on a signal."""
+        if not bot.enabled:
+            self.stdout.write(
+                self.style.WARNING(
+                    'django-aiogram is disabled '
+                    "(TELEGRAM_BOT['ENABLED'] or DJANGO_AIOGRAM_ENABLED); "
+                    'not starting the bot.'
+                )
+            )
+            if options['idle']:
+                self._idle_until_signalled()
+            return
+
+        configured = current_mode()
+        mode = options['mode'] or configured
+        self.stdout.write(f'Updates arrive by {mode}.')
+        if mode != configured:
+            # the webhook view reads the setting, not this flag, so it would
+            # refuse the updates this process is no longer polling for
+            self.stdout.write(
+                self.style.WARNING(
+                    f"--mode {mode} disagrees with TELEGRAM_BOT['MODE'] ({configured}), and it "
+                    'changes this process only: '
+                    + (
+                        'the webhook view still refuses updates while the setting says polling'
+                        if mode == UpdateMode.WEBHOOK
+                        else 'getUpdates fails while a webhook is registered'
+                    )
+                )
+            )
+
+        delivery = get_delivery(handler=bot.send_raw)
+        self._preflight(delivery)
+        # before a thread exists, because `read_timeout()` refuses a non-numeric
+        # `REDIS_TIMEOUT` — and raised from the `finally` below it would skip `close()`,
+        # `collect()` and `recorder.stop()`, stranding the drain's own messages
+        join_timeout = read_timeout() + 1
+        threads: list[threading.Thread] = []
+
+        # Both modes: starting the consumer before the loop runs would let a
+        # backlog reach send_raw while loop.is_running() is still False, so the
+        # coroutine would be driven from the consumer thread. Deferring the start
+        # until the loop picks up this callback keeps the loop single-threaded.
+        # Webhook mode used to start it directly because nothing ran the loop
+        # there — something does now, which is what this change is about.
+        #
+        # Started through a callback on the loop, so it cannot begin before the loop is
+        # turning, and refused once the shutdown starts. close() runs one turn of the loop
+        # on purpose, so a callback still queued when we reach the finally would
+        # start the consumer *after* stop() and after the joins — a thread nobody
+        # waits for, doing Redis work, whose first act is reclaim()
+        shutting_down = threading.Event()
+
+        def start_consuming() -> None:
+            """Start the consumer thread on the loop, unless the shutdown got there first.
+
+            Queued with ``call_soon`` so the thread begins once the loop is turning, and
+            gated because the callback can still be pending when the teardown runs — see
+            the comment above for what an ungated one starts, and how late.
+            """
+            if shutting_down.is_set():
+                logger.info('not starting the consumer: the shutdown had already begun')
+                return
+            threads.append(delivery.start_thread())
+
+        bot.loop.call_soon(start_consuming)
+        previous = self._install_sigterm_handler()
+
+        try:
+            with contextlib.suppress(KeyboardInterrupt, SystemExit):
+                if mode == UpdateMode.WEBHOOK:
+                    self.stdout.write('Consuming the queue; updates are expected over HTTP.')
+                    self._idle_on_the_loop()
+                else:
+                    bot.start_polling()
+        finally:
+            logger.info('shutting down')
+            # before stop(), so the callback above cannot slip a consumer in
+            # behind the joins below
+            shutting_down.set()
+            delivery.stop()
+            for thread in threads:
+                # derived from the bound that actually governs the thread: every
+                # call it makes is capped by REDIS_TIMEOUT, and its blocking pop
+                # by one less than that. BLPOP_TIMEOUT + 1 was six seconds against
+                # a worst case of ten, so a consumer that outlived the join went on
+                # to acknowledge a message close() had already refused
+                thread.join(timeout=join_timeout)
+                if thread.is_alive():
+                    logger.warning(
+                        'the delivery consumer did not stop in time',
+                        extra={'tg_timeout': join_timeout},
+                    )
+            try:
+                bot.close()
+            finally:
+                # the sends close() just drained reported themselves finished into a
+                # queue whose only reader is the consumer loop, and that returned before
+                # the join above — so without this every message the drain delivered
+                # stays in the in-flight list and the next start sends it again. A
+                # graceful stop duplicated whatever the drain had time to finish, which
+                # is the one thing `Delivery.md` says a *kill* is needed for
+                try:
+                    delivery.collect()
+                finally:
+                    # after close(), never before: closing drains in-flight sends,
+                    # and those are what produce the final rows. In its own finally
+                    # because a close() that raises must not also lose the rows
+                    recorder.stop()
+            if previous is not None:
+                # the command may be called in-process; leaving our handler
+                # installed would turn a later SIGTERM into a stray interrupt
+                with contextlib.suppress(ValueError):
+                    signal.signal(signal.SIGTERM, previous)
+
+    def _idle_on_the_loop(self) -> None:
+        """Wait on the bot's loop rather than on an Event.
+
+        In webhook mode this process consumes the queue and nothing drove the
+        loop, so every send the consumer scheduled sat there until something else
+        happened to run it — the next update, or `close()`. `run_forever` is what
+        makes a scheduled send run when it is scheduled, and it unwinds on
+        SIGTERM exactly as `start_polling` does, so the teardown below is
+        unchanged.
+        """
+        stop = self.idle_event or threading.Event()
+        loop = bot.loop
+
+        def wait_then_stop() -> None:
+            """Wait for the idle event on this thread, then stop the loop from it.
+
+            The main thread is inside ``run_forever`` and cannot wait for anything, so
+            the wait lives here and reaches the loop through ``call_soon_threadsafe`` —
+            the only safe way in from another thread. A loop already closed raises
+            ``RuntimeError``, which is a race with the teardown and not a fault.
+            """
+            stop.wait()
+            with contextlib.suppress(RuntimeError):
+                loop.call_soon_threadsafe(loop.stop)
+
+        threading.Thread(target=wait_then_stop, name='tgbot-idle', daemon=True).start()
+        loop.run_forever()
+
+    def _idle_until_signalled(self) -> None:
+        """Hold a disabled container open, and unwind it the way the enabled path does.
+
+        The same SIGTERM handler, so `docker stop` exits 0 rather than 143: without it the
+        signal kills the process outright and a container idling on purpose looked like one
+        that crashed. And `recorder.stop()`, because a disabled process with the log on
+        still has a writer thread holding a database connection.
+        """
+        self.stdout.write('Idling. Send SIGINT or SIGTERM to stop.')
+        previous = self._install_sigterm_handler()
+        try:
+            with contextlib.suppress(KeyboardInterrupt):
+                (self.idle_event or threading.Event()).wait()
+        finally:
+            recorder.stop()
+            if previous is not None:
+                with contextlib.suppress(ValueError):
+                    signal.signal(signal.SIGTERM, previous)
+
+    def _preflight(self, delivery: Delivery) -> None:
+        """Everything worth saying or refusing before a thread exists."""
+        self._warn_about_an_unstable_worker_name()
+        self._require_crash_safety(delivery)
+
+    def _warn_about_an_unstable_worker_name(self) -> None:
+        """Say it here, where being the consumer is known.
+
+        The in-flight list is keyed on the worker's name, so a name that changes when the
+        container is replaced strands whatever the old one was sending. As a system check
+        this can only be information: `manage.py check` runs in every process, and a check
+        cannot tell a consumer from a web tier — as a warning it failed
+        `check --fail-level WARNING` in containers that own no in-flight list at all.
+
+        This process is the consumer. The same rule, reused rather than restated, so the
+        two cannot drift.
+        """
+        from django_aiogram.config.checks import worker_name_problems  # noqa: PLC0415 - no aiogram at import
+
+        for problem in worker_name_problems():
+            logger.warning(
+                'the worker name will not survive a replacement container',
+                extra={'tg_worker': worker_identity()},
+            )
+            self.stdout.write(self.style.WARNING(f'WORKER_NAME {problem.message}'))
+
+    @staticmethod
+    def _require_crash_safety(delivery: Delivery) -> None:
+        """Refuse to start where a killed worker loses the message it was sending.
+
+        Probed here rather than from inside ``run()``: that is a daemon thread, so
+        a ``SystemExit`` raised there kills only the thread and leaves a process
+        polling updates with a dead consumer. A ``CommandError`` gives a non-zero
+        exit and a restart loop somebody can see.
+
+        ``reclaim()`` is the probe, and it is the same call ``run()`` opens with.
+        An unreachable Redis returns False with crash safety still intact, which
+        must not be read as an old server — a blip is not a reason to refuse to
+        start.
+        """
+        if not coerce_bool(conf['REQUIRE_CRASH_SAFE'], f"{SETTINGS_NAME}['REQUIRE_CRASH_SAFE']"):
+            return
+        settled = delivery.reclaim()
+        if delivery.crash_safe:
+            if not settled:
+                # NOPERM and WRONGTYPE come back this way too, and unlike a blip
+                # they do not clear. Refusing here would turn every restart into
+                # a crash loop, so say plainly that the guarantee is unproven
+                # rather than let silence read as a passed check
+                logger.warning(
+                    'could not verify crash-safe delivery: the probe did not settle',
+                    extra={'tg_key': delivery.queue_key},
+                )
+            return
+        msg = (
+            'This Redis predates LMOVE (6.2), so a worker killed mid-send loses that message, '
+            f"and {SETTINGS_NAME}['REQUIRE_CRASH_SAFE'] refuses to run that way. Upgrade the "
+            'server, or set it to False to accept at-most-once delivery.'
+        )
+        raise CommandError(msg)
+
+    @staticmethod
+    def _install_sigterm_handler() -> Handler:
+        """Turn SIGTERM into KeyboardInterrupt so `docker stop` unwinds cleanly.
+
+        Returns the handler it replaced, or None when it could not install one —
+        signal.signal only works on the main thread.
+        """
+
+        def raise_interrupt(_signum: int, _frame: FrameType | None) -> None:
+            """Raise where the signal arrived, which is inside whatever was blocking.
+
+            That is the whole trick: ``KeyboardInterrupt`` unwinds ``start_polling`` and
+            ``run_forever`` through the same path a Ctrl-C takes, so one teardown covers
+            both an operator and ``docker stop``.
+            """
+            raise KeyboardInterrupt
+
+        try:
+            return signal.signal(signal.SIGTERM, raise_interrupt)
+        except ValueError:
+            return None

django_aiogram/management/commands/tgbot_healthcheck.py

@@ -0,0 +1,57 @@
+"""Answer whether the bot container is doing its job.
+
+`docker ps` says the process is up, which is not the same thing: the consumer
+thread can be dead while polling continues, or Redis can be unreachable, and the
+container stays "healthy" either way.
+
+A wrapper, since 3.1.0, over :mod:`django_aiogram.healthcheck`. The decision
+lives there so that ``python -m django_aiogram.healthcheck`` can make it without
+``django.setup()`` — a management command populates the app registry and runs every
+``AppConfig.ready()`` in the host project first, which is 17.89s in one measured
+consumer against 0.01s of actual probing, and more than any Docker ``timeout`` can
+honestly allow. This command is unchanged for anyone who has it in a compose file
+today, and **Deployment** says why the other form belongs in a healthcheck.
+"""
+
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management import BaseCommand, CommandError
+
+from django_aiogram.healthcheck import add_limit_flags, check
+
+
+class Command(BaseCommand):
+    """Check Redis, the consumer's heartbeat and the queue length, in that order."""
+
+    help = 'Exit 0 when the bot container is healthy, non-zero with a reason otherwise'
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        """Declare the two limits, both of which default to a setting.
+
+        Taken from the module that acts on them rather than restated here: a second copy
+        of a flag is how one form ends up with a default the other does not have.
+        """
+        add_limit_flags(parser)
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        """Report the first thing that is wrong, or that everything is fine.
+
+        ``stranded`` and ``guarantee`` are asked for explicitly, because this command's
+        output must not change for anyone who has it in a compose file — while the
+        container-facing entry point leaves both off, since neither can alter the
+        verdict and both are the expensive part of the probe.
+        """
+        report = check(
+            max_queue=options['max_queue'],
+            max_age=options['max_age'],
+            stranded=True,
+            guarantee=True,
+        )
+        if not report.ok:
+            raise CommandError(report.message)
+        # plain when nothing was examined: a disabled process is not a healthy bot, and
+        # this command has never colored that line green
+        self.stdout.write(self.style.SUCCESS(report.message) if report.checked else report.message)
+        for warning in report.warnings:
+            self.stdout.write(self.style.WARNING(warning))

django_aiogram/management/commands/tgbot_prune_events.py

@@ -0,0 +1,144 @@
+"""Delete event log rows older than the retention window.
+
+Nothing on the write path deletes anything, so the table grows until this runs.
+Schedule it; check ``W006`` says so while the retention is unset.
+
+The deletion walks **primary key ranges**, not ``pk__in``. Two reasons, both
+practical: MySQL rejects ``DELETE ... WHERE id IN (SELECT ... LIMIT n)`` against
+the same table, which is exactly what the ORM generates for the obvious
+formulation; and a bounded range at the cold end of the table cannot conflict
+with the inserts still arriving at the hot end, which is what keeps InnoDB's
+next-key locks out of the picture.
+"""
+
+import datetime
+import logging
+import time
+from argparse import ArgumentParser
+from typing import Any, NamedTuple
+
+from django.core.management import BaseCommand, CommandError
+from django.db import connections, models, transaction
+from django.utils import timezone
+
+from django_aiogram.config.settings import conf
+from django_aiogram.eventlog.writer import log_alias
+from django_aiogram.models import TelegramEvent
+
+logger = logging.getLogger('django_aiogram')
+
+
+class Window(NamedTuple):
+    """The range one run walks: the rows, its ends, the cutoff and the alias."""
+
+    rows: models.QuerySet[TelegramEvent]
+    low: int
+    watermark: int
+    cutoff: datetime.datetime
+    alias: str
+
+
+class Command(BaseCommand):
+    """Prune the event log in bounded chunks, one transaction each."""
+
+    help = 'Delete bot event rows older than the retention window'
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        """Declare the window, the chunking and the two safety valves."""
+        parser.add_argument(
+            '--days',
+            type=int,
+            default=None,
+            help="delete rows older than this. Defaults to TELEGRAM_BOT['EVENT_LOG_RETENTION_DAYS'].",
+        )
+        parser.add_argument(
+            '--chunk',
+            type=int,
+            default=1000,
+            help='width of the id range each transaction covers. Rows inside it that are still '
+            'within the window are left alone, so a chunk deletes at most this many (default 1000)',
+        )
+        parser.add_argument(
+            '--sleep',
+            type=float,
+            default=0.1,
+            help='seconds between chunks. The valve for replica lag: with row-based binlogs '
+            'every deleted row is an event (default 0.1)',
+        )
+        parser.add_argument(
+            '--max-chunks',
+            type=int,
+            default=0,
+            help='stop after this many chunks, so a nightly run has a bounded blast radius. 0 means no limit',
+        )
+        parser.add_argument('--database', default=None, help='the alias to prune; defaults to the configured one')
+        parser.add_argument('--dry-run', action='store_true', help='report what would be deleted, and delete nothing')
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        """Walk the table by primary key, deleting one bounded range per commit."""
+        days = options['days'] if options['days'] is not None else int(conf['EVENT_LOG_RETENTION_DAYS'])
+        if days <= 0:
+            self.stdout.write('Retention is not set, so nothing is pruned. See EVENT_LOG_RETENTION_DAYS.')
+            return
+
+        alias = options['database'] or log_alias()
+        if alias not in connections:
+            # E041 guards the setting; the flag bypasses it, in the one command that runs
+            # from cron — where a Django traceback is the least useful thing to wake up to
+            msg = f'no database is configured under the alias {alias!r}; DATABASES has {sorted(connections)}.'
+            raise CommandError(msg)
+        cutoff = timezone.now() - datetime.timedelta(days=days)
+        rows = TelegramEvent.objects.using(alias)
+
+        # where the walk stops: nothing older than the cutoff lives above this id.
+        # `drai_event_recent` covers the cutoff range, so neither form touches the
+        # table — but ordering by id still sorts that range, and `EXPLAIN QUERY PLAN`
+        # gives this and `Min(id)` the same two steps: the covering search and one
+        # `USE TEMP B-TREE FOR ORDER BY`. Written as a limit rather than an aggregate
+        # to read like the `low` below it, not because it measures faster
+        expired = rows.filter(created_at__lt=cutoff)
+        watermark = expired.order_by('-id').values_list('id', flat=True).first()
+        if watermark is None:
+            self.stdout.write(f'Nothing older than {cutoff.isoformat()}.')
+            return
+        # filtered by the cutoff like the watermark is. Taking the table's lowest
+        # id instead meant one surviving row down there pinned the walk to
+        # restart from it every night, and a --max-chunks run never got past it
+        low = expired.order_by('id').values_list('id', flat=True).first() or 1
+
+        deleted = self._walk(Window(rows, low, watermark, cutoff, alias), options)
+        verb = 'would delete' if options['dry_run'] else 'deleted'
+        self.stdout.write(f'{verb} {deleted} events older than {cutoff.isoformat()} from {alias!r}.')
+
+    def _walk(self, window: 'Window', options: dict[str, Any]) -> int:
+        """Delete each id range in turn, pausing between them."""
+        rows, low, watermark, cutoff, alias = window
+        chunk = max(1, int(options['chunk']))
+        limit = max(0, int(options['max_chunks']))
+        pause = max(0.0, float(options['sleep']))
+        deleted = 0
+        rounds = 0
+
+        while low <= watermark:
+            high = min(low + chunk - 1, watermark)
+            # the id range is the access path; created_at is the correctness
+            # condition, because id order only approximates time order once
+            # several processes and a buffered writer are involved
+            batch = rows.filter(id__gte=low, id__lte=high, created_at__lt=cutoff)
+            if options['dry_run']:
+                removed_here = batch.count()
+            else:
+                with transaction.atomic(using=alias):
+                    removed_here, _ = batch.delete()
+            deleted += removed_here
+            low = high + 1
+            rounds += 1
+            if limit and rounds >= limit:
+                self.stdout.write(f'Stopped after {rounds} chunks; rerun to continue.')
+                break
+            # nothing was deleted, so there is nothing for a replica to catch up
+            # on and nothing to vacuum; a dry run deletes nothing at all
+            if pause and removed_here and low <= watermark and not options['dry_run']:
+                # replicas and autovacuum both need the gaps
+                time.sleep(pause)
+        return deleted

django_aiogram/management/commands/tgbot_reclaim.py

@@ -0,0 +1,135 @@
+"""Put a dead worker's in-flight messages back on the queue.
+
+A message being sent lives in ``<queue>:processing:<worker>`` until the send
+finishes, and the worker that put it there reclaims it on its next start. That
+only works while the name is stable — a container with no ``hostname:`` and no
+``WORKER_NAME`` gets a fresh one every time, and its messages are stranded where
+nothing will ever look for them again.
+
+This is the way out, and it is deliberately manual: naming the dead worker is a
+human saying it is dead. Nothing here probes for liveness, because a worker that
+is merely slow looks exactly like one that is gone, and taking a message back
+from a live sender is how you send it twice.
+"""
+
+import logging
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management import BaseCommand, CommandError
+from redis import Redis
+from redis.exceptions import ResponseError
+
+from django_aiogram.eventlog.events import worker_identity
+from django_aiogram.redis import get_redis, processing_key, queue_key
+
+logger = logging.getLogger('django_aiogram')
+
+
+class Command(BaseCommand):
+    """Move one worker's in-flight messages back to the queue."""
+
+    help = 'Requeue the messages a dead worker left in flight'
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        """Declare the worker to reclaim from, and the safety valves."""
+        parser.add_argument(
+            '--worker',
+            required=True,
+            help='the WORKER_NAME (or hostname) whose in-flight list to drain. Naming it is you '
+            'saying that worker is gone: reclaiming from a live one sends its message twice.',
+        )
+        parser.add_argument(
+            '--limit',
+            type=int,
+            default=0,
+            help='stop after this many messages, so one run has a bounded blast radius. 0 means no '
+            'limit. A bounded run takes the newest in flight first, because that is the end of the '
+            'list a reclaim pops from',
+        )
+        parser.add_argument('--dry-run', action='store_true', help='report what is there, and move nothing')
+
+    @staticmethod
+    def _move_one(connection: Redis, source: str, destination: str) -> object:
+        """Move the newest in-flight message back to the front of the queue.
+
+        Newest, because a message is taken with ``LEFT`` → ``RIGHT`` and so the tail of the
+        in-flight list is the most recent one — and this pops that tail. Draining the whole
+        list therefore restores the original order, the oldest ending up at the front; a run
+        stopped by ``--limit`` has reclaimed the newest and left the older ones in place.
+
+        ``LMOVE ... RIGHT LEFT`` is what ``reclaim()`` uses, so the order a real run
+        produces is the order a dry run promises. On a Redis older than 6.2 that command
+        does not exist, and this is the one path that can still recover those messages:
+        the consumer's own ``reclaim()`` gives up there and runs at-most-once, so a list
+        stranded before the downgrade would have nothing else to come back through.
+        ``RPOPLPUSH`` is the same move, and has been there since 1.2.
+        """
+        try:
+            return connection.lmove(source, destination, 'RIGHT', 'LEFT')
+        except ResponseError as error:
+            if 'unknown command' not in str(error).lower():
+                raise
+            return connection.rpoplpush(source, destination)
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        """Walk the named worker's in-flight list back onto the queue."""
+        worker = str(options['worker']).strip()
+        if not worker:
+            msg = '--worker cannot be empty.'
+            raise CommandError(msg)
+        limit = int(options['limit'])
+        if limit < 0:
+            # max(0, ...) would have read this as "no limit", which is the
+            # opposite of what someone typing a limit is asking for. Judged with
+            # the other arguments, so --dry-run reports the mistake rather than
+            # returning happily on a run that would have been refused
+            msg = f'--limit cannot be negative, got {limit}. Use 0 for no limit.'
+            raise CommandError(msg)
+        if worker == worker_identity():
+            # this process would be reclaiming from whatever is running here now,
+            # which on a bot container is the consumer that is mid-send
+            msg = (
+                f"{worker!r} is this process's own worker name. A running consumer reclaims its own "
+                'messages when it starts; taking them from underneath it sends them twice.'
+            )
+            raise CommandError(msg)
+
+        source, destination = processing_key(worker), queue_key()
+        connection = get_redis()
+        try:
+            waiting = int(connection.llen(source) or 0)
+        except Exception as error:
+            msg = f'could not read {source}: {error}'
+            raise CommandError(msg) from error
+
+        if not waiting:
+            self.stdout.write(f'Nothing in flight for {worker!r}.')
+            return
+        if options['dry_run']:
+            # through the same limit the real run applies. A rehearsal that
+            # promises to move two and then moves one is worse than none: it is
+            # read as the plan, and the difference shows up as messages left
+            # behind that nobody went looking for
+            would_move = min(waiting, limit) if limit else waiting
+            self.stdout.write(
+                f'{waiting} message(s) in flight for {worker!r}; would requeue {would_move} of them.'
+                if would_move != waiting
+                else f'{waiting} message(s) in flight for {worker!r}; would requeue them.'
+            )
+            return
+
+        moved = 0
+        while not limit or moved < limit:
+            try:
+                # `is None`, not falsy: nil is how both commands say the source list is
+                # empty, and a payload that happens to be empty is still a message
+                if self._move_one(connection, source, destination) is None:
+                    break
+            except Exception as error:
+                msg = f'moved {moved} message(s), then failed: {error}'
+                raise CommandError(msg) from error
+            moved += 1
+
+        logger.info('reclaimed a dead worker', extra={'tg_key': source, 'tg_count': moved})
+        self.stdout.write(self.style.SUCCESS(f'Requeued {moved} message(s) from {worker!r}.'))