adc-streaming 2.5.0__tar.gz → 2.6.0__tar.gz

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: adc-streaming
-Version: 2.5.0
+Version: 2.6.0
 Summary: Astronomy Data Commons streaming client libraries
 Home-page: https://github.com/astronomy-commons/adc-streaming
 Author: Astronomy Data Commons Team
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS :: MacOS X
 Description-Content-Type: text/markdown
-Provides-Extra: dev
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.11.0
+Requires-Dist: dataclasses; python_version < "3.7"
+Requires-Dist: importlib-metadata; python_version < "3.8"
+Requires-Dist: tqdm
+Requires-Dist: certifi>=2020.04.05.1
+Requires-Dist: typing-extensions; python_version < "3.8"
+Provides-Extra: dev
+Requires-Dist: autopep8; extra == "dev"
+Requires-Dist: docker; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
+Requires-Dist: pytest-integration; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx_rtd_theme; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 # Astronomy Data Commons Streaming Client Libraries
 

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/adc/auth.py

@@ -64,6 +64,7 @@ class SASLAuth(object):
             self._config = {
                 "security.protocol": "SASL_SSL",
                 "ssl.ca.location": ssl_cert,
+                "https.ca.location": ssl_cert,
             }
             if "ssl_endpoint_identification_algorithm" in kwargs:
                 self._config["ssl.endpoint.identification.algorithm"] = \

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/adc/consumer.py

@@ -14,7 +14,6 @@ import confluent_kafka.admin  # type: ignore
 
 from .auth import SASLAuth
 from .errors import ErrorCallback, log_client_errors
-from .oidc import set_oauth_cb
 
 
 class LogicalOffset(enum.IntEnum):
@@ -360,5 +359,4 @@ class ConsumerConfig:
 
         if self.auth is not None:
             config.update(self.auth())
-        set_oauth_cb(config)
         return config

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/adc/producer.py

@@ -14,7 +14,6 @@ import confluent_kafka  # type: ignore
 from .auth import SASLAuth
 from .errors import (DeliveryCallback, ErrorCallback, log_client_errors,
                      log_delivery_errors)
-from .oidc import set_oauth_cb
 
 
 class Producer:
@@ -48,11 +47,26 @@ class Producer:
                                 "Either configure a topic when constructing the Producer, "
                                 "or specify the topic argument to write()")
         self.logger.debug("writing message to %s", topic)
+        produce_kwargs = {"headers": headers, "key": key}
         if delivery_callback is not None:
-            self._producer.produce(topic, msg, headers=headers, key=key,
-                                   on_delivery=delivery_callback)
-        else:
-            self._producer.produce(topic, msg, headers=headers, key=key,)
+            produce_kwargs["on_delivery"] = delivery_callback
+        while True:
+            try:
+                self._producer.produce(topic, msg, **produce_kwargs)
+                break
+            except BufferError:
+                # It's hard to know what the size limit on the buffer is, so we will try to
+                # wait until the number of items in it decreases (or it is empty, in case all
+                # messages were successfully sent in the time it took to handle the error).
+                buffer_len = len(self._producer)
+                self.logger.debug(f"Blocking due to BufferError, buffer size: {buffer_len}")
+                while buffer_len > 0 and len(self._producer) == buffer_len:
+                    self._producer.poll(0.01)
+
+    def queued_message_count(self):
+        """Get the number of messages waiting to be sent to the broker.
+        """
+        return len(self._producer)
 
     def flush(self, timeout: timedelta = timedelta(seconds=10)) -> int:
         """Attempt to flush enqueued messages. Return the number of messages still
@@ -66,9 +80,9 @@ class Producer:
             self.logger.debug("flushed all messages")
         return n
 
-    def close(self) -> int:
+    def close(self, timeout: timedelta = timedelta(seconds=10)) -> int:
         self.logger.debug("shutting down producer")
-        return self.flush()
+        return self.flush(timeout)
 
     def __enter__(self) -> 'Producer':
         return self
@@ -142,7 +156,6 @@ class ProducerConfig:
             config["error_cb"] = self.error_callback
         if self.auth is not None:
             config.update(self.auth())
-        set_oauth_cb(config)
         return config
 
 

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/adc_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: adc-streaming
-Version: 2.5.0
+Version: 2.6.0
 Summary: Astronomy Data Commons streaming client libraries
 Home-page: https://github.com/astronomy-commons/adc-streaming
 Author: Astronomy Data Commons Team
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS :: MacOS X
 Description-Content-Type: text/markdown
-Provides-Extra: dev
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.11.0
+Requires-Dist: dataclasses; python_version < "3.7"
+Requires-Dist: importlib-metadata; python_version < "3.8"
+Requires-Dist: tqdm
+Requires-Dist: certifi>=2020.04.05.1
+Requires-Dist: typing-extensions; python_version < "3.8"
+Provides-Extra: dev
+Requires-Dist: autopep8; extra == "dev"
+Requires-Dist: docker; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
+Requires-Dist: pytest-integration; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx_rtd_theme; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 # Astronomy Data Commons Streaming Client Libraries
 

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/adc_streaming.egg-info/SOURCES.txt

@@ -12,7 +12,6 @@ adc/consumer.py
 adc/errors.py
 adc/io.py
 adc/kafka.py
-adc/oidc.py
 adc/producer.py
 adc_streaming.egg-info/PKG-INFO
 adc_streaming.egg-info/SOURCES.txt

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/adc_streaming.egg-info/requires.txt

@@ -1,6 +1,4 @@
-authlib
-confluent-kafka!=2.1.0,!=2.1.1,>=1.6.1
-requests
+confluent-kafka>=2.11.0
 tqdm
 certifi>=2020.04.05.1
 

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/setup.py

@@ -3,11 +3,9 @@ from setuptools import setup
 
 # requirements
 install_requires = [
-    "authlib",  # FIXME: drop after next release of confluent-kafka with OIDC support
-    "confluent-kafka >= 1.6.1, != 2.1.0, != 2.1.1",
+    "confluent-kafka >= 2.11.0",
     "dataclasses ; python_version < '3.7'",
     "importlib-metadata ; python_version < '3.8'",
-    "requests",  # FIXME: drop after next release of confluent-kafka with OIDC support
     "tqdm",
     "certifi>=2020.04.05.1",
     "typing-extensions ; python_version < '3.8'",

{adc-streaming-2.5.0 → adc_streaming-2.6.0}/tests/test_kafka_integration.py

@@ -84,6 +84,44 @@ class KafkaIntegrationTestCase(unittest.TestCase):
         self.assertEqual(msg.value(), b"can you hear me?")
         self.assertEqual(msg.key(), b"test_msg")
 
+    def test_message_with_callback(self):
+        """Try writing a message into the Kafka broker, with a delivery callback.
+        """
+        topic = "test_message_with_callback"
+        callback_called = False
+        callback_error = False
+
+        def callback(err, msg):
+            nonlocal callback_called
+            nonlocal callback_error
+            callback_called = True
+            callback_error = err is not None
+
+        ap = adc.producer
+        with ap.Producer(ap.ProducerConfig(broker_urls=[self.kafka.address],
+                                           topic=topic, auth=self.kafka.auth)) as producer:
+            producer.write("message data", delivery_callback=callback)
+            producer.flush()
+            consumer = adc.consumer.Consumer(adc.consumer.ConsumerConfig(
+                broker_urls=[self.kafka.address],
+                group_id="test_consumer",
+                auth=self.kafka.auth,
+            ))
+            consumer.subscribe(topic)
+            stream = consumer.stream()
+
+            msg = next(stream)
+            if msg.error() is not None:
+                raise Exception(msg.error())
+            # give the producer's background thread time to notice the ack from the broker and
+            # fire the callback
+            producer._producer.poll(0.1)
+
+        self.assertEqual(msg.topic(), topic)
+        self.assertEqual(msg.value(), b"message data")
+        self.assertEqual(callback_called, True)
+        self.assertEqual(callback_error, False)
+
     def test_reset_to_end(self):
         # Write a few messages.
         topic = "test_reset_to_end"
@@ -486,7 +524,7 @@ class KafkaDockerConnection:
         """Block until the Docker daemon tells us the IP and Port of the Kafa broker.
 
         Returns the ip and port as a string in the form "ip:port."
-"""
+        """
         i = 0
         while (not self.query_kafka_broker_address()) and i < maxiter:
             logger.info("polling to wait for container to acquire port...")
@@ -508,7 +546,7 @@ class KafkaDockerConnection:
         port = addrs[0]['HostPort']
         return f"{ip}:{port}"
 
-    def poll_for_kafka_active(self, maxiter=20, sleep=timedelta(milliseconds=500)):
+    def poll_for_kafka_active(self, maxiter=40, sleep=timedelta(milliseconds=500)):
         """Block until Kafka's network listener is accepting connections."""
         i = 0
         while (not self.query_kafka_active()) and i < maxiter:

adc-streaming-2.5.0/adc/oidc.py

@@ -1,27 +0,0 @@
-def set_oauth_cb(config):
-    """Implement client support for KIP-768 OpenID Connect.
-
-    Apache Kafka 3.1.0 supports authentication using OpenID Client Credentials.
-    Native support for Python is still incomplete due to this issue:
-    https://github.com/confluentinc/librdkafka/issues/3751
-
-    Meanwhile, this is a pure Python implementation of the refresh token
-    callback.
-    """
-    if config.pop('sasl.oauthbearer.method', None) != 'oidc':
-        return
-
-    client_id = config.pop('sasl.oauthbearer.client.id')
-    client_secret = config.pop('sasl.oauthbearer.client.secret')
-    scope = config.pop('sasl.oauthbearer.scope', None)
-    token_endpoint = config.pop('sasl.oauthbearer.token.endpoint.url')
-
-    from authlib.integrations.requests_client import OAuth2Session
-    session = OAuth2Session(client_id, client_secret, scope=scope)
-
-    def oauth_cb(*_, **__):
-        token = session.fetch_token(
-            token_endpoint, grant_type='client_credentials')
-        return token['access_token'], token['expires_at']
-
-    config['oauth_cb'] = oauth_cb