nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/ingest/consumer/consumer.py

@@ -25,8 +25,9 @@ from typing import Optional, Union
 import backoff
 import nats
 import nats.js.api
+import nats.js.errors
 from nats.aio.client import Msg
-from nucliadb_protos.writer_pb2 import BrokerMessage, BrokerMessageBlobReference
+from nats.js import JetStreamContext
 
 from nucliadb.common.cluster.exceptions import ShardsNotFound
 from nucliadb.common.maindb.driver import Driver
@@ -34,16 +35,18 @@ from nucliadb.common.maindb.exceptions import ConflictError
 from nucliadb.ingest import logger
 from nucliadb.ingest.orm.exceptions import DeadletteredError, SequenceOrderViolation
 from nucliadb.ingest.orm.processor import Processor, sequence_manager
+from nucliadb_protos.writer_pb2 import BrokerMessage, BrokerMessageBlobReference
 from nucliadb_telemetry import context, errors, metrics
 from nucliadb_utils import const
 from nucliadb_utils.cache.pubsub import PubSubDriver
 from nucliadb_utils.nats import MessageProgressUpdater, NatsConnectionManager
 from nucliadb_utils.settings import nats_consumer_settings
 from nucliadb_utils.storages.storage import Storage
+from nucliadb_utils.utilities import has_feature
 
 consumer_observer = metrics.Observer(
     "message_processor",
-    labels={"source": ""},
+    labels={"source": "", "partition": ""},
     buckets=[
         0.01,
         0.025,
@@ -83,40 +86,62 @@ class IngestConsumer:
 
         self.lock = lock or asyncio.Lock()
         self.processor = Processor(driver, storage, pubsub, partition)
+        self.subscription: Optional[JetStreamContext.PullSubscription] = None
+
+    async def ack_message(self, msg: Msg, kbid: Optional[str] = None):
+        context = {}
+        if kbid:
+            context["kbid"] = kbid
+        if has_feature(const.Features.NATS_SYNC_ACK, default=False, context=context):
+            await msg.ack_sync(timeout=10)
+        else:
+            await msg.ack()
 
     async def initialize(self):
         await self.setup_nats_subscription()
         self.initialized = True
 
+    async def finalize(self):
+        if self.initialized:
+            await self.teardown_nats_subscription()
+            self.initialized = False
+
+    async def teardown_nats_subscription(self):
+        if self.subscription is not None:
+            try:
+                await self.nats_connection_manager.unsubscribe(self.subscription)
+            except nats.errors.ConnectionClosedError:
+                logger.warning("Connection closed while unsubscribing")
+                pass
+            self.subscription = None
+
     async def setup_nats_subscription(self):
         last_seqid = await sequence_manager.get_last_seqid(self.driver, self.partition)
         if last_seqid is None:
             last_seqid = 1
         subject = const.Streams.INGEST.subject.format(partition=self.partition)
-        await self.nats_connection_manager.subscribe(
-            subject=subject,
-            queue=const.Streams.INGEST.group.format(partition=self.partition),
+        durable_name = const.Streams.INGEST.group.format(partition=self.partition)
+        self.subscription = await self.nats_connection_manager.pull_subscribe(
             stream=const.Streams.INGEST.name,
-            flow_control=True,
+            subject=subject,
+            durable=durable_name,
             cb=self.subscription_worker,
             subscription_lost_cb=self.setup_nats_subscription,
             config=nats.js.api.ConsumerConfig(
+                durable_name=durable_name,
                 deliver_policy=nats.js.api.DeliverPolicy.BY_START_SEQUENCE,
                 opt_start_seq=last_seqid,
                 ack_policy=nats.js.api.AckPolicy.EXPLICIT,
-                max_ack_pending=nats_consumer_settings.nats_max_ack_pending,
+                max_ack_pending=1,
                 max_deliver=nats_consumer_settings.nats_max_deliver,
                 ack_wait=nats_consumer_settings.nats_ack_wait,
-                idle_heartbeat=nats_consumer_settings.nats_idle_heartbeat,
             ),
         )
         logger.info(
-            f"Subscribed to {subject} on stream {const.Streams.INGEST.name} from {last_seqid}"
+            f"Subscribed pull consumer to {subject} on stream {const.Streams.INGEST.name} from {last_seqid}"
         )
 
-    @backoff.on_exception(
-        backoff.expo, (ConflictError,), jitter=backoff.random_jitter, max_tries=4
-    )
+    @backoff.on_exception(backoff.expo, (ConflictError,), jitter=backoff.random_jitter, max_tries=4)
     async def _process(self, pb: BrokerMessage, seqid: int):
         await self.processor.process(pb, seqid, self.partition)
 
@@ -142,18 +167,29 @@ class IngestConsumer:
                 logger.warning("Could not delete blob reference", exc_info=True)
 
     async def subscription_worker(self, msg: Msg):
+        kbid: Optional[str] = None
         subject = msg.subject
         reply = msg.reply
         seqid = int(reply.split(".")[5])
         message_source = "<msg source not set>"
+        num_delivered = msg.metadata.num_delivered
+        if num_delivered > 1:
+            logger.warning(
+                "Message has been redelivered",
+                extra={
+                    "seqid": seqid,
+                    "subject": subject,
+                    "reply": reply,
+                    "num_delivered": num_delivered,
+                },
+            )
         start = time.monotonic()
 
-        async with MessageProgressUpdater(
-            msg, nats_consumer_settings.nats_ack_wait * 0.66
-        ), self.lock:
-            logger.info(
-                f"Message processing: subject:{subject}, seqid: {seqid}, reply: {reply}"
-            )
+        async with (
+            MessageProgressUpdater(msg, nats_consumer_settings.nats_ack_wait * 0.66),
+            self.lock,
+        ):
+            logger.info(f"Message processing: subject:{subject}, seqid: {seqid}, reply: {reply}")
             try:
                 pb = await self.get_broker_message(msg)
                 if pb.source == pb.MessageSource.PROCESSOR:
@@ -169,36 +205,39 @@ class IngestConsumer:
                     f"Received from {message_source} on {pb.kbid}/{pb.uuid} seq {seqid} partition {self.partition} at {time}"  # noqa
                 )
                 context.add_context({"kbid": pb.kbid, "rid": pb.uuid})
-
+                kbid = pb.kbid
                 try:
-                    with consumer_observer(
-                        {
-                            "source": "writer"
-                            if pb.source == pb.MessageSource.WRITER
-                            else "processor"
-                        }
-                    ):
+                    source = "writer" if pb.source == pb.MessageSource.WRITER else "processor"
+                    with consumer_observer({"source": source, "partition": self.partition}):
                         await self._process(pb, seqid)
                 except SequenceOrderViolation as err:
-                    log_func = logger.error
-                    if seqid == err.last_seqid:  # pragma: no cover
-                        # Occasional retries of the last processed message may happen
-                        log_func = logger.warning
-                    log_func(
-                        f"Old txn: DISCARD (nucliadb seqid: {seqid}, partition: {self.partition}). Current seqid: {err.last_seqid}"  # noqa
+                    logger.log(
+                        level=logging.ERROR if seqid < err.last_seqid else logging.WARNING,
+                        msg="Old txn. Discarding message",
+                        extra={
+                            "stored_seqid": err.last_seqid,
+                            "message_seqid": seqid,
+                            "partition": self.partition,
+                            "kbid": pb.kbid,
+                            "msg_delivered_count": msg.metadata.num_delivered,
+                        },
                     )
                 else:
                     message_type_name = pb.MessageType.Name(pb.type)
                     time_to_process = time.monotonic() - start
-                    log_level = (
-                        logging.INFO if time_to_process < 10 else logging.WARNING
-                    )
+                    log_level = logging.INFO if time_to_process < 10 else logging.WARNING
                     logger.log(
                         log_level,
-                        f"Successfully processed {message_type_name} message from \
-                            {message_source}. kb: {pb.kbid}, resource: {pb.uuid}, \
-                                nucliadb seqid: {seqid}, partition: {self.partition} as {audit_time}, \
-                                    total time: {time_to_process:.2f}s",
+                        f"Successfully processed {message_type_name} message",
+                        extra={
+                            "kbid": pb.kbid,
+                            "rid": pb.uuid,
+                            "message_source": message_source,
+                            "nucliadb_seqid": seqid,
+                            "partition": self.partition,
+                            "total_time": time_to_process,
+                            "audit_time": audit_time,
+                        },
                     )
             except DeadletteredError as e:
                 # Messages that have been sent to deadletter at some point
@@ -209,7 +248,8 @@ class IngestConsumer:
                     f"A copy of the message has been stored on {self.processor.storage.deadletter_bucket}. "
                     f"Check sentry for more details: {str(e)}"
                 )
-                await msg.ack()
+                await self.ack_message(msg, kbid)
+                logger.info("Message acked because of deadletter", extra={"seqid": seqid})
             except (ShardsNotFound,) as e:
                 # Any messages that for some unexpected inconsistency have failed and won't be tried again
                 # as we cannot do anything about it
@@ -220,7 +260,8 @@ class IngestConsumer:
                     f"This message has been dropped and won't be retried again"
                     f"Check sentry for more details: {str(e)}"
                 )
-                await msg.ack()
+                await self.ack_message(msg, kbid)
+                logger.info("Message acked because of drop", extra={"seqid": seqid})
             except Exception as e:
                 # Unhandled exceptions that need to be retried after a small delay
                 errors.capture_exception(e)
@@ -230,10 +271,12 @@ class IngestConsumer:
                     f"Check sentry for more details: {str(e)}"
                 )
                 await msg.nak()
+                logger.info("Message nacked because of unhandled error", extra={"seqid": seqid})
                 raise e
             else:
                 # Successful processing
-                await msg.ack()
+                await self.ack_message(msg, kbid)
+                logger.info("Message acked because of success", extra={"seqid": seqid})
                 await self.clean_broker_message(msg)
 
 
@@ -250,28 +293,29 @@ class IngestProcessedConsumer(IngestConsumer):
 
     async def setup_nats_subscription(self):
         subject = const.Streams.INGEST_PROCESSED.subject
-        await self.nats_connection_manager.subscribe(
-            subject=subject,
-            queue=const.Streams.INGEST_PROCESSED.group,
+        durable_name = const.Streams.INGEST_PROCESSED.group
+        self.subscription = await self.nats_connection_manager.pull_subscribe(
             stream=const.Streams.INGEST_PROCESSED.name,
-            flow_control=True,
+            subject=subject,
+            durable=durable_name,
             cb=self.subscription_worker,
             subscription_lost_cb=self.setup_nats_subscription,
             config=nats.js.api.ConsumerConfig(
+                durable_name=durable_name,
                 ack_policy=nats.js.api.AckPolicy.EXPLICIT,
-                max_ack_pending=100,  # custom ack pending here
+                deliver_policy=nats.js.api.DeliverPolicy.ALL,
+                # We set it to 20 because we don't care about order here and we want to be able to HPA based
+                # on the number of pending messages in the queue.
+                max_ack_pending=20,
                 max_deliver=nats_consumer_settings.nats_max_deliver,
                 ack_wait=nats_consumer_settings.nats_ack_wait,
-                idle_heartbeat=nats_consumer_settings.nats_idle_heartbeat,
             ),
         )
         logger.info(
-            f"Subscribed to {subject} on stream {const.Streams.INGEST_PROCESSED.name}"
+            f"Subscribed pull consumer to {subject} on stream {const.Streams.INGEST_PROCESSED.name}"
         )
 
-    @backoff.on_exception(
-        backoff.expo, (ConflictError,), jitter=backoff.random_jitter, max_tries=4
-    )
+    @backoff.on_exception(backoff.expo, (ConflictError,), jitter=backoff.random_jitter, max_tries=4)
     async def _process(self, pb: BrokerMessage, seqid: int):
         """
         We are setting `transaction_check` to False here because we can not mix

nucliadb/ingest/consumer/materializer.py

@@ -29,6 +29,7 @@ from nucliadb_protos import writer_pb2
 from nucliadb_utils import const
 from nucliadb_utils.cache.pubsub import PubSubDriver
 from nucliadb_utils.storages.storage import Storage
+from nucliadb_utils.utilities import get_audit
 
 from .utils import DelayedTaskHandler
 
@@ -82,25 +83,22 @@ class MaterializerHandler:
         notification.ParseFromString(data)
 
         if (
-            notification.action
-            != writer_pb2.Notification.Action.COMMIT  # only on commits
+            notification.action != writer_pb2.Notification.Action.COMMIT  # only on commits
             or notification.write_type
             == writer_pb2.Notification.WriteType.MODIFIED  # only on new resources and deletes
         ):
             return
 
-        self.task_handler.schedule(
-            notification.kbid, partial(self.process, notification.kbid)
-        )
+        self.task_handler.schedule(notification.kbid, partial(self.process, notification.kbid))
 
     async def process(self, kbid: str) -> None:
         logger.info(f"Materializing knowledgebox", extra={"kbid": kbid})
-        async with datamanagers.with_transaction(read_only=True) as txn:
-            value = await datamanagers.resources.calculate_number_of_resources(
-                txn, kbid=kbid
-            )
+        async with datamanagers.with_ro_transaction() as txn:
+            value = await datamanagers.resources.calculate_number_of_resources(txn, kbid=kbid)
         async with datamanagers.with_transaction() as txn:
-            await datamanagers.resources.set_number_of_resources(
-                txn, kbid=kbid, value=value
-            )
+            await datamanagers.resources.set_number_of_resources(txn, kbid=kbid, value=value)
             await txn.commit()
+
+        audit = get_audit()
+        if audit:
+            audit.report_resources(kbid=kbid, resources=value)

nucliadb/ingest/consumer/pull.py

@@ -21,10 +21,7 @@ import asyncio
 import base64
 from typing import Optional
 
-import nats
-import nats.errors
 from aiohttp.client_exceptions import ClientConnectorError
-from nucliadb_protos.writer_pb2 import BrokerMessage, BrokerMessageBlobReference
 
 from nucliadb.common import datamanagers
 from nucliadb.common.http_clients.processing import ProcessingHTTPClient, get_nua_api_id
@@ -32,11 +29,13 @@ from nucliadb.common.maindb.driver import Driver
 from nucliadb.ingest import logger, logger_activity
 from nucliadb.ingest.orm.exceptions import ReallyStopPulling
 from nucliadb.ingest.orm.processor import Processor
+from nucliadb_protos.writer_pb2 import BrokerMessage, BrokerMessageBlobReference
 from nucliadb_telemetry import errors
 from nucliadb_utils import const
 from nucliadb_utils.cache.pubsub import PubSubDriver
 from nucliadb_utils.settings import nuclia_settings
 from nucliadb_utils.storages.storage import Storage
+from nucliadb_utils.transaction import MaxTransactionSizeExceededError
 from nucliadb_utils.utilities import get_storage, get_transaction_utility
 
 
@@ -78,9 +77,7 @@ class PullWorker:
         data = base64.b64decode(payload)
         pb.ParseFromString(data)
 
-        logger.debug(
-            f"Resource: {pb.uuid} KB: {pb.kbid} ProcessingID: {pb.processing_id}"
-        )
+        logger.debug(f"Resource: {pb.uuid} KB: {pb.kbid} ProcessingID: {pb.processing_id}")
 
         if not self.local_subscriber:
             transaction_utility = get_transaction_utility()
@@ -93,11 +90,9 @@ class PullWorker:
                     # send to separate processor
                     target_subject=const.Streams.INGEST_PROCESSED.subject,
                 )
-            except nats.errors.MaxPayloadError:
+            except MaxTransactionSizeExceededError:
                 storage = await get_storage()
-                stored_key = await storage.set_stream_message(
-                    kbid=pb.kbid, rid=pb.uuid, data=data
-                )
+                stored_key = await storage.set_stream_message(kbid=pb.kbid, rid=pb.uuid, data=data)
                 referenced_pb = BrokerMessageBlobReference(
                     uuid=pb.uuid, kbid=pb.kbid, storage_key=stored_key
                 )
@@ -141,9 +136,7 @@ class PullWorker:
             try:
                 pull_type_id = get_nua_api_id()
             except Exception as exc:
-                logger.exception(
-                    "Could not read NUA API Key. Can not start pull worker"
-                )
+                logger.exception("Could not read NUA API Key. Can not start pull worker")
                 raise ReallyStopPulling() from exc
         else:
             pull_type_id = "main"
@@ -152,7 +145,7 @@ class PullWorker:
             logger.info(f"Collecting from NucliaDB Cloud {self.partition} partition")
             while True:
                 try:
-                    async with datamanagers.with_transaction() as txn:
+                    async with datamanagers.with_ro_transaction() as txn:
                         cursor = await datamanagers.processing.get_pull_offset(
                             txn, pull_type_id=pull_type_id, partition=self.partition
                         )
@@ -176,9 +169,7 @@ class PullWorker:
                                 await self.handle_message(payload)
                         except Exception as e:
                             errors.capture_exception(e)
-                            logger.exception(
-                                "Error while pulling and processing message/s"
-                            )
+                            logger.exception("Error while pulling and processing message/s")
                             raise e
                         async with datamanagers.with_transaction() as txn:
                             await datamanagers.processing.set_pull_offset(
@@ -189,9 +180,7 @@ class PullWorker:
                             )
                             await txn.commit()
                     elif data.status == "empty":
-                        logger_activity.debug(
-                            f"No messages waiting in partition #{self.partition}"
-                        )
+                        logger_activity.debug(f"No messages waiting in partition #{self.partition}")
                         await asyncio.sleep(self.pull_time_empty_backoff)
                     else:
                         logger.info(f"Proxy pull answered with error: {data}")
@@ -202,9 +191,7 @@ class PullWorker:
                     KeyboardInterrupt,
                     SystemExit,
                 ):
-                    logger.info(
-                        f"Pull task for partition #{self.partition} was canceled, exiting"
-                    )
+                    logger.info(f"Pull task for partition #{self.partition} was canceled, exiting")
                     raise ReallyStopPulling()
 
                 except ClientConnectorError:
@@ -214,14 +201,12 @@ class PullWorker:
                     )
                     await asyncio.sleep(self.pull_time_error_backoff)
 
-                except nats.errors.MaxPayloadError as e:
+                except MaxTransactionSizeExceededError as e:
                     if data is not None:
                         payload_length = 0
                         if data.payload:
                             payload_length = len(base64.b64decode(data.payload))
-                        logger.error(
-                            f"Message too big for transaction: {payload_length}"
-                        )
+                        logger.error(f"Message too big for transaction: {payload_length}")
                     raise e
                 except Exception:
                     logger.exception("Unhandled error pulling messages from processing")

nucliadb/ingest/consumer/service.py

@@ -45,9 +45,7 @@ from .shard_creator import ShardCreatorHandler
 def _handle_task_result(task: asyncio.Task) -> None:
     e = task.exception()
     if e:
-        logger.exception(
-            "Loop stopped by exception. This should not happen. Exiting.", exc_info=e
-        )
+        logger.exception("Loop stopped by exception. This should not happen. Exiting.", exc_info=e)
         sys.exit(1)
 
 
@@ -87,9 +85,7 @@ async def start_ingest_consumers(
     if transaction_settings.transaction_local:
         raise ConfigurationError("Can not start ingest consumers in local mode")
 
-    while len(
-        manager.get_index_nodes()
-    ) == 0 and running_settings.running_environment not in (
+    while len(manager.get_index_nodes()) == 0 and running_settings.running_environment not in (
         "local",
         "test",
     ):
@@ -101,9 +97,9 @@ async def start_ingest_consumers(
     storage = await get_storage(service_name=service_name or SERVICE_NAME)
     nats_connection_manager = get_nats_manager()
 
-    max_concurrent_processing = asyncio.Semaphore(
-        settings.max_concurrent_ingest_processing
-    )
+    max_concurrent_processing = asyncio.Semaphore(settings.max_concurrent_ingest_processing)
+
+    consumer_finalizers = []
 
     for partition in settings.partitions:
         consumer = IngestConsumer(
@@ -115,8 +111,15 @@ async def start_ingest_consumers(
             lock=max_concurrent_processing,
         )
         await consumer.initialize()
+        consumer_finalizers.append(consumer.finalize)
 
-    return nats_connection_manager.finalize
+    async def _finalize():
+        # Finalize all the consumers and the nats connection manager
+        for consumer_finalize in consumer_finalizers:
+            await consumer_finalize()
+        await nats_connection_manager.finalize()
+
+    return _finalize
 
 
 async def start_ingest_processed_consumer(
@@ -132,9 +135,7 @@ async def start_ingest_processed_consumer(
     if transaction_settings.transaction_local:
         raise ConfigurationError("Can not start ingest consumers in local mode")
 
-    while len(
-        manager.get_index_nodes()
-    ) == 0 and running_settings.running_environment not in (
+    while len(manager.get_index_nodes()) == 0 and running_settings.running_environment not in (
         "local",
         "test",
     ):
@@ -159,22 +160,22 @@ async def start_ingest_processed_consumer(
 
 
 async def start_auditor() -> Callable[[], Awaitable[None]]:
-    driver = await setup_driver()
     audit = get_audit()
     assert audit is not None
+
     pubsub = await get_pubsub()
     assert pubsub is not None, "Pubsub is not configured"
     storage = await get_storage(service_name=SERVICE_NAME)
-    index_auditor = IndexAuditHandler(driver=driver, audit=audit, pubsub=pubsub)
-    resource_writes_auditor = ResourceWritesAuditHandler(
-        driver=driver, storage=storage, audit=audit, pubsub=pubsub
-    )
+    index_auditor = IndexAuditHandler(audit=audit, pubsub=pubsub)
+    resource_writes_auditor = ResourceWritesAuditHandler(storage=storage, audit=audit, pubsub=pubsub)
 
     await index_auditor.initialize()
     await resource_writes_auditor.initialize()
 
     return partial(
-        asyncio.gather, index_auditor.finalize(), resource_writes_auditor.finalize()  # type: ignore
+        asyncio.gather,
+        index_auditor.finalize(),
+        resource_writes_auditor.finalize(),  # type: ignore
     )
 
 

nucliadb/ingest/consumer/shard_creator.py

@@ -22,7 +22,7 @@ import logging
 import uuid
 from functools import partial
 
-from nucliadb.common import datamanagers, locking
+from nucliadb.common import locking
 from nucliadb.common.cluster.manager import choose_node
 from nucliadb.common.cluster.utils import get_shard_manager
 from nucliadb.common.maindb.driver import Driver
@@ -82,21 +82,19 @@ class ShardCreatorHandler:
             metrics.total_messages.inc({"type": "shard_creator", "action": "ignored"})
             return
 
-        self.task_handler.schedule(
-            notification.kbid, partial(self.process_kb, notification.kbid)
-        )
+        self.task_handler.schedule(notification.kbid, partial(self.process_kb, notification.kbid))
         metrics.total_messages.inc({"type": "shard_creator", "action": "scheduled"})
 
     @metrics.handler_histo.wrap({"type": "shard_creator"})
     async def process_kb(self, kbid: str) -> None:
         logger.info({"message": "Processing notification for kbid", "kbid": kbid})
         async with self.driver.transaction(read_only=True) as txn:
-            kb_shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
             current_shard = await self.shard_manager.get_current_active_shard(txn, kbid)
 
-        if kb_shards is None or current_shard is None:
+        if current_shard is None:
             logger.error(
-                "Processing a notification for a nonexistent", extra={"kbid": kbid}
+                "Processing a notification for KB with no current shard",
+                extra={"kbid": kbid},
             )
             return
 
@@ -105,13 +103,8 @@ class ShardCreatorHandler:
         async with locking.distributed_lock(locking.NEW_SHARD_LOCK.format(kbid=kbid)):
             # remember, a lock will do at least 1+ reads and 1 write.
             # with heavy writes, this adds some simple k/v pressure
-            node, shard_id = choose_node(current_shard)
+            node, shard_id = choose_node(current_shard, use_nidx=True)
             shard: nodereader_pb2.Shard = await node.reader.GetShard(
                 nodereader_pb2.GetShardRequest(shard_id=noderesources_pb2.ShardId(id=shard_id))  # type: ignore
             )
-            await self.shard_manager.maybe_create_new_shard(
-                kbid,
-                shard.paragraphs,
-                shard.fields,
-                kb_shards.release_channel,
-            )
+            await self.shard_manager.maybe_create_new_shard(kbid, shard.paragraphs)

nucliadb/ingest/consumer/utils.py

@@ -48,9 +48,7 @@ class DelayedTaskHandler:
         for task in list(self.outstanding_tasks.values()):
             await task
 
-    def schedule(
-        self, key: str, handler: Callable[[], Coroutine[None, None, None]]
-    ) -> None:
+    def schedule(self, key: str, handler: Callable[[], Coroutine[None, None, None]]) -> None:
         if key in self.to_process:
             # already waiting to process this key, ignore
             return