DIRAC 9.0.26__tar.gz → 9.0.28__tar.gz

{dirac-9.0.26 → dirac-9.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.26
+Version: 9.0.28
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.26
+Requires-Dist: DIRACCommon==v9.0.28
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1
@@ -28,7 +28,7 @@ Requires-Dist: fts3
 Requires-Dist: gfal2-python
 Requires-Dist: importlib_metadata>=4.4
 Requires-Dist: importlib_resources
-Requires-Dist: M2Crypto>=0.36
+Requires-Dist: M2Crypto>=0.50
 Requires-Dist: packaging
 Requires-Dist: pexpect
 Requires-Dist: prompt-toolkit>=3

{dirac-9.0.26 → dirac-9.0.28}/dirac.cfg

@@ -1160,6 +1160,15 @@ Operations
       pilotVORepoBranch = master # Branch to use
       workDir = /tmp/pilot3Files # Local work directory on the masterCS for synchronisation
     }
+    # Installation specific help shown by dirac-proxy-init when a proxy can't be created
+    # Use "\n" for line breaks as values can't span multiple lines
+    ProxyInit
+    {
+      # Shown when the certificate DN is not registered in DIRAC
+      NotRegisteredMessage = New users must join the VO, see:\n    https://example.com/join-the-vo
+      # Shown when the VOMS server refuses to give the user an attribute certificate
+      VOMSFailureMessage = Check your certificate is linked to your account in IAM, see:\n    https://example.com/link-certificate
+    }
     Services
     {
       # See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html

{dirac-9.0.26 → dirac-9.0.28}/environment.yml

@@ -21,15 +21,7 @@ dependencies:
   - fts3
   - gitpython >=2.1.0
   - invoke
-  # Upper pin: m2crypto 0.46.0 rewrote ssl_read() in src/SWIG/_ssl.i and inverted the
-  # handling of ssl_sleep_with_timeout()'s return value (and dropped the gettimeofday()
-  # that initialises the deadline). Any read() on a connection with a timeout set now
-  # busy-loops and returns with a stale SSLTimeoutError set, which surfaces as
-  # "SystemError: <built-in function ssl_read> returned a result with an exception set"
-  # and breaks all DIRAC DISET/dips connections. Still unfixed upstream as of 0.49.0.
-  ## Reported in
-  ## https://lists.sr.ht/~mcepl/m2crypto/%3C7b94d866-bdd2-4a29-9713-b6a6fd076ad7@cta-observatory.org%3E
-  - m2crypto >=0.43,<0.46
+  - m2crypto >=0.50
   - matplotlib
   - numpy
   - pexpect >=4.0.1

{dirac-9.0.26 → dirac-9.0.28}/release.notes

@@ -1,3 +1,56 @@
+[v9.0.28]
+
+*WorkloadManagement
+
+FIX: (#8851) set the job Status and LastUpdateTime atomically so that the StalledJobAgent does not reschedule jobs that have just been matched
+FIX: (#8851) StalledJobAgent re-checks stuck jobs before rescheduling them and records itself as the source
+FIX: (#8838) the payload no longer inherits signals blocked in the JobWrapper (e.g. by gfal2), so it can be stopped with SIGTERM and abort itself with SIGQUIT
+NEW: (#8829) allow fractional MatchingDelay for sub-second matcher throttling
+NEW: (#8829) MatchingDelay is a token bucket, with an optional MatchingDelay/<Site>/Burst/<Attribute>/<Value> burst size
+FIX: (#8829) MatchingDelay tokens are reserved before matching, so concurrent requests can no longer exceed the configured rate
+FIX: (#8829) running limits count the jobs matched since the cached running counts were taken, so sites no longer overshoot them
+FIX: (#8829) CEs and Burst subsections are no longer reported as unknown job attributes
+
+*DataManagementSystem
+
+FIX: (#8848) Add permission check to FC rebuildDirectoryUsage
+FIX: (#8836) S3Gateway presigned URLs work with S3 implementations which validate them, and store the checksum on upload
+
+*FrameworkSystem
+
+FIX: (#8845) Increase token key size
+FIX: (#8829) MessageQueueHandler sends log records from a background thread with a bounded queue, so logging never blocks when the broker is slow or unreachable
+
+*RequestManagementSystem
+
+FIX: (#8833) The RequestExecutingAgent no longer crashes repeatedly on requests without any operation to execute, it sets an error and delays them by a day
+CHANGE: (#8833) Request and Operation constructors no longer modify the dictionary they are given
+
+*Core
+
+FIX: (#8829) Add wait timeout when connecting to stomp
+FIX: (#8822) annotate DIRAC.exit as NoReturn so pylint knows DIRACExit ends a branch
+NEW: (#8818) Improved error messages when dirac-proxy-init fails
+FIX: (#8782) require m2crypto >=0.50, which restores the timed ssl_read() retries broken in 0.46.0
+
+*Resources
+
+FIX: (#8829) StompMQConnector.put fails fast (one short reconnection attempt, with backoff) instead of blocking for minutes when the broker is unreachable
+
+*tests
+
+CHANGE: (#8828) diracx-init-db sets a URL for every DiracX DB, initialising only the DiracX-owned ones in MySQL
+
+*ResourceStatusSystem
+
+CHANGE: (#8777) Remove some of SpaceTokenOccupancyCache methods from ResourceManagementClient/Handler
+
+[v9.0.27]
+
+*WorkloadManagement
+
+FIX: (#8809) stop sending JobType and JobGroup as job parameters (duplicate)
+
 [v9.0.26]
 
 *MonitoringSystem

{dirac-9.0.26 → dirac-9.0.28}/setup.cfg

@@ -26,7 +26,7 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.26
+	DIRACCommon==v9.0.28
 	diracx-client >=v0.0.1
 	diracx-core >=v0.0.1
 	diracx-cli >=v0.0.1
@@ -35,7 +35,7 @@ install_requires =
 	gfal2-python
 	importlib_metadata >=4.4
 	importlib_resources
-	M2Crypto >=0.36
+	M2Crypto >=0.50
 	packaging
 	pexpect
 	prompt-toolkit >=3
@@ -160,6 +160,7 @@ console_scripts =
 	dirac-admin-sysadmin-cli = DIRAC.FrameworkSystem.scripts.dirac_admin_sysadmin_cli:main [admin]
 	dirac-admin-update-instance = DIRAC.FrameworkSystem.scripts.dirac_admin_update_instance:main [admin]
 	dirac-admin-update-pilot = DIRAC.FrameworkSystem.scripts.dirac_admin_update_pilot:main [admin]
+	dirac-admin-debug-user = DIRAC.FrameworkSystem.scripts.dirac_admin_debug_user:main
 	dirac-admin-users-with-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_users_with_proxy:main [admin]
 	dirac-install-component = DIRAC.FrameworkSystem.scripts.dirac_install_component:main [server]
 	dirac-proxy-destroy = DIRAC.FrameworkSystem.scripts.dirac_proxy_destroy:main

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/Core/DISET/ThreadConfig.py

@@ -131,7 +131,7 @@ def threadDeco(method):
 
         :return: wrapped method
         """
-        deco = tc.getDecorator()
+        deco = tc.getDecorator()  # pylint: disable=no-member
         if not deco:
             return method(*args, **kwargs)
         # Deco is a decorator sooo....

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/Core/DISET/private/BaseClient.py

@@ -1,6 +1,7 @@
-""" This module exposes the BaseClient class,
-    which serves as base for InnerRPCClient and TransferClient.
+"""This module exposes the BaseClient class,
+which serves as base for InnerRPCClient and TransferClient.
 """
+
 import time
 
 import _thread
@@ -243,8 +244,10 @@ class BaseClient:
             self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS]
 
         # Are we delegating something?
-        delegatedDN = self.kwargs.get(self.KW_DELEGATED_DN) or self.__threadConfig.getDN()
-        delegatedGroup = self.kwargs.get(self.KW_DELEGATED_GROUP) or self.__threadConfig.getGroup()
+        delegatedDN = self.kwargs.get(self.KW_DELEGATED_DN) or self.__threadConfig.getDN()  # pylint: disable=no-member
+        delegatedGroup = (
+            self.kwargs.get(self.KW_DELEGATED_GROUP) or self.__threadConfig.getGroup()  # pylint: disable=no-member
+        )
         if delegatedDN:
             self.kwargs[self.KW_DELEGATED_DN] = delegatedDN
             if not delegatedGroup:

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py

@@ -259,7 +259,7 @@ class TornadoBaseClient:
         if self.KW_EXTRA_CREDENTIALS in self.kwargs:
             self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS]
         # Are we delegating something?
-        delegatedDN, delegatedGroup = self.__threadConfig.getID()
+        delegatedDN, delegatedGroup = self.__threadConfig.getID()  # pylint: disable=no-member
         if self.KW_DELEGATED_DN in self.kwargs and self.kwargs[self.KW_DELEGATED_DN]:
             delegatedDN = self.kwargs[self.KW_DELEGATED_DN]
         elif delegatedDN:

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py

@@ -1057,11 +1057,16 @@ class FileCatalogDB(DB):
 
         return S_OK({"Successful": successful, "Failed": failed})
 
-    def rebuildDirectoryUsage(self):
+    def rebuildDirectoryUsage(self, credDict={}):
         """Rebuild DirectoryUsage table from scratch"""
 
-        result = self.dtree._rebuildDirectoryUsage()
-        return result
+        result = self._checkAdminPermission(credDict)
+        if not result["OK"]:
+            return result
+        if not result["Value"]:
+            return S_ERROR(errno.EACCES, "Not authorized to rebuild directory usage")
+
+        return self.dtree._rebuildDirectoryUsage()
 
     def repairCatalog(self, credDict={}):
         """Repair catalog inconsistencies"""

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/DataManagementSystem/Service/FileCatalogHandler.py

@@ -378,7 +378,7 @@ class FileCatalogHandlerMixin:
 
     def export_rebuildDirectoryUsage(self):
         """Rebuild DirectoryUsage table from scratch"""
-        return self.fileCatalogDB.rebuildDirectoryUsage()
+        return self.fileCatalogDB.rebuildDirectoryUsage(self.getRemoteCredentials())
 
     types_repairCatalog = []
 

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py

@@ -115,8 +115,8 @@ class S3GatewayHandlerMixin:
             # If we can't obtain remote credentials, consider it permission denied
             return S_ERROR(errno.EACCES, "Could not obtain remote credentials")
 
-        self._tc.setDN(credDict["DN"])
-        self._tc.setGroup(credDict["group"])
+        self._tc.setDN(credDict["DN"])  # pylint: disable=no-member
+        self._tc.setGroup(credDict["group"])  # pylint: disable=no-member
 
         successful = {}
         failed = {}
@@ -150,9 +150,9 @@ class S3GatewayHandlerMixin:
                     failed[url] = "Permission denied"
                     continue
 
-                res = returnSingleResult(
-                    s3Plugin.createPresignedUrl({url: urls.get("Fields")}, s3_method, expiration=expiration)
-                )
+                # For put_object, urls maps each url to the fields (metadata) of the file
+                fields = urls[url] if s3_method == "put_object" else None
+                res = returnSingleResult(s3Plugin.createPresignedUrl({url: fields}, s3_method, expiration=expiration))
 
                 log.debug(f"Presigned URL for {url}: {res}")
                 if res["OK"]:

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/FrameworkSystem/DB/AuthDB.py

@@ -154,7 +154,7 @@ class AuthDB(SQLAlchemyDB):
 
         :return: S_OK/S_ERROR
         """
-        key = JsonWebKey.generate_key("RSA", 1024, is_private=True)
+        key = JsonWebKey.generate_key("RSA", 2048, is_private=True)
         # as_dict has no arguments for authlib < 1.0.0
         # for authlib >= 1.0.0
         keyDict = dict(

{dirac-9.0.26 → dirac-9.0.28}/src/DIRAC/FrameworkSystem/private/authorization/AuthServer.py

@@ -397,7 +397,7 @@ class AuthServer(_AuthorizationServer):
         """
         try:
             response = super().create_authorization_response(response, username)
-            response.clear_cookie("auth_session")
+            response.clear_cookie("auth_session")  # pylint: disable=no-member
             return response
         except Exception as e:
             sLog.exception(e)

dirac-9.0.28/src/DIRAC/FrameworkSystem/private/standardLogging/Handler/MessageQueueHandler.py

@@ -0,0 +1,188 @@
+"""
+Message Queue Handler
+"""
+import json
+import logging
+import os
+import queue as _queue
+import socket
+import sys
+import threading
+import time
+
+from DIRAC.Resources.MessageQueue.MQCommunication import createProducer
+
+
+class MessageQueueHandler(logging.Handler):
+    """
+    MessageQueueHandler is a custom handler from logging.
+    It has no equivalent in the standard logging library because it is linked to DIRAC.
+
+    It is useful to send log messages to a destination, like the StreamHandler to a stream, the FileHandler to a file.
+    Here, this handler send log messages to a message queue server.
+
+    Sending is done by a background thread: emit() only formats the record and puts it in a
+    bounded in-memory queue, so that logging never blocks the caller, even when the broker is
+    slow or unreachable. When the queue is full, or a record cannot be sent, the record is
+    dropped and counted, and the number of dropped records is reported on stderr.
+
+    There is an assumption made that the formatter used is JsonFormatter
+    """
+
+    #: Maximum number of records waiting to be sent
+    MAX_QUEUE_SIZE = 10_000
+    #: Seconds to pause sending after a failure, doubled up to the maximum while failures persist
+    RETRY_BACKOFF_INITIAL = 1
+    RETRY_BACKOFF_MAX = 60
+    #: Minimum seconds between two reports of dropped records
+    DROP_REPORT_INTERVAL = 60
+    #: Seconds given to the sender to flush the queue when the handler is closed
+    CLOSE_TIMEOUT = 5
+
+    def __init__(self, queue, maxQueueSize=None):
+        """
+        Initialization of the MessageQueueHandler.
+
+        :param queue: queue identifier in the configuration.
+                      example: "mardirac3.in2p3.fr::Queues::TestQueue"
+        :param int maxQueueSize: maximum number of records waiting to be sent
+        """
+        super().__init__()
+        self.hostname = socket.gethostname()
+        self._mqURI = queue
+        self._maxQueueSize = maxQueueSize or self.MAX_QUEUE_SIZE
+        self._droppedLock = threading.Lock()
+        self._forkLock = threading.Lock()
+        self._dropped = 0
+        self._lastDropReport = 0.0
+        self._startSender()
+
+    def _startSender(self):
+        """Create the queue and start the sender thread (again, in a forked child)"""
+        # The producer is created by the sender thread, as connecting may take a while
+        self.producer = None
+        self._records = _queue.Queue(maxsize=self._maxQueueSize)
+        self._stopping = threading.Event()
+        self._backoff = 0
+        self._sender = threading.Thread(target=self._run, name="MessageQueueHandler", daemon=True)
+        self._sender.start()
+        # Set last, so that other threads of a forked child only see the new pid once all is ready
+        self._pid = os.getpid()
+
+    def emit(self, record):
+        """
+        Add the record to the queue of records to be sent. Never blocks.
+
+        :param record: log record object
+        """
+        # Records emitted while sending (e.g. errors from the MQ connector itself) are not
+        # sent to the broker: during an outage they would feed failures back into the queue.
+        if threading.current_thread() is self._sender:
+            return
+        # Threads do not survive a fork: the child needs its own sender
+        if os.getpid() != self._pid:
+            with self._forkLock:
+                if os.getpid() != self._pid:
+                    self._startSender()
+        try:
+            # add the hostname to the record
+            record.hostname = self.hostname
+            strRecord = self.format(record)
+        except Exception:
+            self.handleError(record)
+            return
+        try:
+            self._records.put_nowait(strRecord)
+        except _queue.Full:
+            self._recordDropped()
+
+    def handle(self, record):
+        """
+        Conditionally emit the specified logging record.
+
+        Override the handle method from logging.Handler as there is no need to
+        acquire the lock to emit the record.
+        """
+        rv = self.filter(record)
+        if rv:
+            self.emit(record)
+        return rv
+
+    def close(self):
+        """Give the sender a short time to flush the queue, then stop it"""
+        self._stopping.set()
+        # Wake the sender up once it has sent everything queued before (if there is room)
+        try:
+            self._records.put_nowait(None)
+        except _queue.Full:
+            pass
+        if self._sender.is_alive() and self._sender is not threading.current_thread():
+            self._sender.join(self.CLOSE_TIMEOUT)
+        super().close()
+
+    @property
+    def dropped(self):
+        """Number of records dropped so far"""
+        return self._dropped
+
+    def _recordDropped(self, count=1):
+        with self._droppedLock:
+            self._dropped += count
+            now = time.monotonic()
+            if now - self._lastDropReport < self.DROP_REPORT_INTERVAL:
+                return
+            self._lastDropReport = now
+            dropped = self._dropped
+        # Not logged: that would add records to the queue being dropped
+        print(f"WARNING MessageQueueHandler: {dropped} log records dropped so far for {self._mqURI}", file=sys.stderr)
+
+    def _run(self):
+        """Body of the sender thread"""
+        while True:
+            try:
+                strRecord = self._records.get(timeout=1)
+            except _queue.Empty:
+                if self._stopping.is_set():
+                    return
+                continue
+            if strRecord is None:
+                return
+            if not self._send(strRecord):
+                self._recordDropped()
+                # Pause before the next attempt; when stopping, give up on what is left
+                if self._stopping.wait(self._backoff):
+                    self._drain()
+                    return
+
+    def _send(self, strRecord):
+        """Send one record, creating the producer if needed
+
+        :returns: True if the record was sent
+        """
+        try:
+            if self.producer is None:
+                result = createProducer(self._mqURI)
+                if result["OK"]:
+                    self.producer = result["Value"]
+            if self.producer is not None:
+                result = self.producer.put(json.loads(strRecord))
+        except Exception as e:
+            result = {"OK": False, "Message": repr(e)}
+        if result["OK"]:
+            self._backoff = 0
+            return True
+        self._backoff = min(max(2 * self._backoff, self.RETRY_BACKOFF_INITIAL), self.RETRY_BACKOFF_MAX)
+        return False
+
+    def _drain(self):
+        """Count the records left in the queue as dropped"""
+        count = 0
+        while True:
+            try:
+                strRecord = self._records.get_nowait()
+            except _queue.Empty:
+                break
+            if strRecord is not None:
+                count += 1
+        if count:
+            self._recordDropped(count)

dirac-9.0.28/src/DIRAC/FrameworkSystem/private/standardLogging/test/Test_Logging_MessageQueueHandler.py

@@ -0,0 +1,167 @@
+"""Tests for the MessageQueueHandler: logging must never block on the message queue"""
+
+import json
+import logging
+import threading
+import time
+from unittest import mock
+
+import pytest
+
+from DIRAC.FrameworkSystem.private.standardLogging.Handler import MessageQueueHandler as mqhModule
+from DIRAC.FrameworkSystem.private.standardLogging.Handler.MessageQueueHandler import MessageQueueHandler
+
+S_OK = {"OK": True, "Value": None}
+
+
+class FakeProducer:
+    """Records what is put, optionally blocking or failing"""
+
+    def __init__(self, block=None, fail=False):
+        self.block = block
+        self.fail = fail
+        self.messages = []
+        self.calls = 0
+
+    def put(self, msg):
+        self.calls += 1
+        if self.block is not None:
+            self.block.wait(10)
+        if self.fail:
+            return {"OK": False, "Message": "broker unreachable"}
+        self.messages.append(msg)
+        return S_OK
+
+
+def makeHandler(monkeypatch, producer, **kwargs):
+    # The producer is created by the sender thread, so the patch must outlive the constructor
+    monkeypatch.setattr(mqhModule, "createProducer", mock.Mock(return_value={"OK": True, "Value": producer}))
+    handler = MessageQueueHandler("mq.example::Queues::Test", **kwargs)
+    handler.setFormatter(logging.Formatter('{"message": "%(message)s"}'))
+    return handler
+
+
+def makeRecord(msg="hello"):
+    return logging.LogRecord("test", logging.INFO, __file__, 1, msg, None, None)
+
+
+def waitFor(condition, timeout=5):
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if condition():
+            return True
+        time.sleep(0.01)
+    return False
+
+
+@pytest.fixture(autouse=True)
+def fastBackoff(monkeypatch):
+    monkeypatch.setattr(MessageQueueHandler, "RETRY_BACKOFF_INITIAL", 0.01)
+    monkeypatch.setattr(MessageQueueHandler, "RETRY_BACKOFF_MAX", 0.02)
+    monkeypatch.setattr(MessageQueueHandler, "CLOSE_TIMEOUT", 1)
+
+
+def test_records_are_sent_in_the_background(monkeypatch):
+    producer = FakeProducer()
+    handler = makeHandler(monkeypatch, producer)
+
+    handler.handle(makeRecord("one"))
+    handler.handle(makeRecord("two"))
+
+    assert waitFor(lambda: len(producer.messages) == 2)
+    assert [m["message"] for m in producer.messages] == ["one", "two"]
+    handler.close()
+
+
+def test_emit_does_not_block_when_the_broker_hangs(monkeypatch):
+    release = threading.Event()
+    producer = FakeProducer(block=release)
+    handler = makeHandler(monkeypatch, producer)
+
+    start = time.monotonic()
+    for i in range(100):
+        handler.handle(makeRecord(f"msg {i}"))
+    assert time.monotonic() - start < 1
+
+    release.set()
+    assert waitFor(lambda: len(producer.messages) == 100)
+    handler.close()
+
+
+def test_records_are_dropped_when_the_queue_is_full(monkeypatch, capsys):
+    release = threading.Event()
+    producer = FakeProducer(block=release)
+    handler = makeHandler(monkeypatch, producer, maxQueueSize=5)
+
+    for i in range(20):
+        handler.handle(makeRecord(f"msg {i}"))
+
+    # One record is being sent (blocked), 5 are queued, the rest is dropped
+    assert waitFor(lambda: handler.dropped >= 14)
+    assert "log records dropped" in capsys.readouterr().err
+    release.set()
+    handler.close()
+
+
+def test_failed_records_are_dropped_and_sending_backs_off(monkeypatch):
+    producer = FakeProducer(fail=True)
+    handler = makeHandler(monkeypatch, producer)
+
+    handler.handle(makeRecord("lost"))
+
+    assert waitFor(lambda: handler.dropped == 1)
+    assert handler._backoff > 0
+
+    # Sending resumes once the broker is back
+    producer.fail = False
+    handler.handle(makeRecord("kept"))
+    assert waitFor(lambda: [m["message"] for m in producer.messages] == ["kept"])
+    assert handler._backoff == 0
+    handler.close()
+
+
+def test_records_from_the_sender_thread_are_not_queued(monkeypatch):
+    producer = FakeProducer()
+    handler = makeHandler(monkeypatch, producer)
+
+    def logFromSender(msg):
+        # e.g. the MQ connector logging an error while sending
+        handler.handle(makeRecord("from the sender"))
+        return FakeProducer.put(producer, msg)
+
+    producer.put = logFromSender
+    handler.handle(makeRecord("original"))
+
+    assert waitFor(lambda: len(producer.messages) == 1)
+    time.sleep(0.1)
+    assert [m["message"] for m in producer.messages] == ["original"]
+    handler.close()
+
+
+def test_producer_creation_happens_in_the_background_and_is_retried(monkeypatch):
+    producer = FakeProducer()
+    results = [{"OK": False, "Message": "no broker"}, {"OK": True, "Value": producer}]
+    createProducer = mock.Mock(side_effect=results)
+    monkeypatch.setattr(mqhModule, "createProducer", createProducer)
+    handler = MessageQueueHandler("mq.example::Queues::Test")
+    handler.setFormatter(logging.Formatter('{"message": "%(message)s"}'))
+    handler.handle(makeRecord("first"))
+    handler.handle(makeRecord("second"))
+
+    assert waitFor(lambda: [m["message"] for m in producer.messages] == ["second"])
+    assert createProducer.call_count == 2
+    assert handler.dropped == 1
+    handler.close()
+
+
+def test_close_flushes_pending_records(monkeypatch):
+    producer = FakeProducer()
+    handler = makeHandler(monkeypatch, producer)
+
+    for i in range(10):
+        handler.handle(makeRecord(f"msg {i}"))
+    handler.close()
+
+    assert len(producer.messages) == 10
+    assert not handler._sender.is_alive()
+    assert json.dumps(producer.messages[0])