adc-streaming 2.4.0__tar.gz → 2.6.0__tar.gz

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/.github/workflows/build.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - name: Check out the code

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: adc-streaming
-Version: 2.4.0
+Version: 2.6.0
 Summary: Astronomy Data Commons streaming client libraries
 Home-page: https://github.com/astronomy-commons/adc-streaming
 Author: Astronomy Data Commons Team
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS :: MacOS X
 Description-Content-Type: text/markdown
-Provides-Extra: dev
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.11.0
+Requires-Dist: dataclasses; python_version < "3.7"
+Requires-Dist: importlib-metadata; python_version < "3.8"
+Requires-Dist: tqdm
+Requires-Dist: certifi>=2020.04.05.1
+Requires-Dist: typing-extensions; python_version < "3.8"
+Provides-Extra: dev
+Requires-Dist: autopep8; extra == "dev"
+Requires-Dist: docker; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
+Requires-Dist: pytest-integration; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx_rtd_theme; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 # Astronomy Data Commons Streaming Client Libraries
 

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/adc/auth.py

@@ -38,6 +38,8 @@ class SASLAuth(object):
     ssl_ca_location : `str`, optional
         If using SSL via a self-signed cert, a path/location
         to the certificate.
+    ssl_endpoint_identification_algorithm : `str`, optional
+        If using SSL, the algorithm used to verify that certificate is valid for the endpoint.
     token_endpoint : `str`, optional
         The OpenID Connect token endpoint URL.
         Required for OAUTHBEARER / OpenID Connect, otherwise ignored.
@@ -62,7 +64,11 @@ class SASLAuth(object):
             self._config = {
                 "security.protocol": "SASL_SSL",
                 "ssl.ca.location": ssl_cert,
+                "https.ca.location": ssl_cert,
             }
+            if "ssl_endpoint_identification_algorithm" in kwargs:
+                self._config["ssl.endpoint.identification.algorithm"] = \
+                    kwargs["ssl_endpoint_identification_algorithm"]
         else:
             self._config = {"security.protocol": "SASL_PLAINTEXT"}
 

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/adc/consumer.py

@@ -14,7 +14,6 @@ import confluent_kafka.admin  # type: ignore
 
 from .auth import SASLAuth
 from .errors import ErrorCallback, log_client_errors
-from .oidc import set_oauth_cb
 
 
 class LogicalOffset(enum.IntEnum):
@@ -38,7 +37,8 @@ class Consumer:
         self.logger = logging.getLogger("adc-streaming.consumer")
         self.conf = conf
         self._consumer = confluent_kafka.Consumer(conf._to_confluent_kafka())
-        # Workaround for https://github.com/confluentinc/librdkafka/issues/3753#issuecomment-1058272987.
+        # Workaround for
+        # https://github.com/confluentinc/librdkafka/issues/3753#issuecomment-1058272987.
         # FIXME: Remove once fixed upstream, or on removal of oauth_cb.
         self._consumer.poll(0)
         self._stop_event = threading.Event()
@@ -196,7 +196,7 @@ class Consumer:
                             self.mark_done(m, asynchronous=True)
                         yield m
                     else:
-                        raise(confluent_kafka.KafkaException(err))
+                        raise (confluent_kafka.KafkaException(err))
             finally:
                 if autocommit:
                     self._consumer.commit(asynchronous=True)
@@ -242,7 +242,7 @@ class Consumer:
                             # Done with all partitions for the topic, remove it
                             del active_partitions[m.topic()]
                     else:
-                        raise(confluent_kafka.KafkaException(err))
+                        raise (confluent_kafka.KafkaException(err))
             finally:
                 if autocommit:
                     self._consumer.commit(asynchronous=True)
@@ -359,5 +359,4 @@ class ConsumerConfig:
 
         if self.auth is not None:
             config.update(self.auth())
-        set_oauth_cb(config)
         return config

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/adc/producer.py

@@ -14,7 +14,6 @@ import confluent_kafka  # type: ignore
 from .auth import SASLAuth
 from .errors import (DeliveryCallback, ErrorCallback, log_client_errors,
                      log_delivery_errors)
-from .oidc import set_oauth_cb
 
 
 class Producer:
@@ -27,7 +26,8 @@ class Producer:
         self.conf = conf
         self.logger.debug(f"connecting to producer with config {conf._to_confluent_kafka()}")
         self._producer = confluent_kafka.Producer(conf._to_confluent_kafka())
-        # Workaround for https://github.com/confluentinc/librdkafka/issues/3753#issuecomment-1058272987.
+        # Workaround for
+        # https://github.com/confluentinc/librdkafka/issues/3753#issuecomment-1058272987.
         # FIXME: Remove once fixed upstream, or on removal of oauth_cb.
         self._producer.poll(0)
 
@@ -35,7 +35,8 @@ class Producer:
               msg: Union[bytes, 'Serializable'],
               headers: Optional[Union[dict, list]] = None,
               delivery_callback: Optional[DeliveryCallback] = log_delivery_errors,
-              topic: Optional[str] = None) -> None:
+              topic: Optional[str] = None,
+              key: Optional[Union[str, bytes]] = None) -> None:
         if isinstance(msg, Serializable):
             msg = msg.serialize()
         if topic is None:
@@ -46,11 +47,26 @@ class Producer:
                                 "Either configure a topic when constructing the Producer, "
                                 "or specify the topic argument to write()")
         self.logger.debug("writing message to %s", topic)
+        produce_kwargs = {"headers": headers, "key": key}
         if delivery_callback is not None:
-            self._producer.produce(topic, msg, headers=headers,
-                                   on_delivery=delivery_callback)
-        else:
-            self._producer.produce(topic, msg, headers=headers)
+            produce_kwargs["on_delivery"] = delivery_callback
+        while True:
+            try:
+                self._producer.produce(topic, msg, **produce_kwargs)
+                break
+            except BufferError:
+                # It's hard to know what the size limit on the buffer is, so we will try to
+                # wait until the number of items in it decreases (or it is empty, in case all
+                # messages were successfully sent in the time it took to handle the error).
+                buffer_len = len(self._producer)
+                self.logger.debug(f"Blocking due to BufferError, buffer size: {buffer_len}")
+                while buffer_len > 0 and len(self._producer) == buffer_len:
+                    self._producer.poll(0.01)
+
+    def queued_message_count(self):
+        """Get the number of messages waiting to be sent to the broker.
+        """
+        return len(self._producer)
 
     def flush(self, timeout: timedelta = timedelta(seconds=10)) -> int:
         """Attempt to flush enqueued messages. Return the number of messages still
@@ -64,9 +80,9 @@ class Producer:
             self.logger.debug("flushed all messages")
         return n
 
-    def close(self) -> int:
+    def close(self, timeout: timedelta = timedelta(seconds=10)) -> int:
         self.logger.debug("shutting down producer")
-        return self.flush()
+        return self.flush(timeout)
 
     def __enter__(self) -> 'Producer':
         return self
@@ -140,7 +156,6 @@ class ProducerConfig:
             config["error_cb"] = self.error_callback
         if self.auth is not None:
             config.update(self.auth())
-        set_oauth_cb(config)
         return config
 
 

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/adc_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: adc-streaming
-Version: 2.4.0
+Version: 2.6.0
 Summary: Astronomy Data Commons streaming client libraries
 Home-page: https://github.com/astronomy-commons/adc-streaming
 Author: Astronomy Data Commons Team
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS :: MacOS X
 Description-Content-Type: text/markdown
-Provides-Extra: dev
 License-File: LICENSE
+Requires-Dist: confluent-kafka>=2.11.0
+Requires-Dist: dataclasses; python_version < "3.7"
+Requires-Dist: importlib-metadata; python_version < "3.8"
+Requires-Dist: tqdm
+Requires-Dist: certifi>=2020.04.05.1
+Requires-Dist: typing-extensions; python_version < "3.8"
+Provides-Extra: dev
+Requires-Dist: autopep8; extra == "dev"
+Requires-Dist: docker; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
+Requires-Dist: pytest-integration; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx_rtd_theme; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 # Astronomy Data Commons Streaming Client Libraries
 

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/adc_streaming.egg-info/SOURCES.txt

@@ -12,7 +12,6 @@ adc/consumer.py
 adc/errors.py
 adc/io.py
 adc/kafka.py
-adc/oidc.py
 adc/producer.py
 adc_streaming.egg-info/PKG-INFO
 adc_streaming.egg-info/SOURCES.txt

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/adc_streaming.egg-info/requires.txt

@@ -1,6 +1,4 @@
-authlib
-confluent-kafka!=2.1.0,!=2.1.1,>=1.6.1
-requests
+confluent-kafka>=2.11.0
 tqdm
 certifi>=2020.04.05.1
 

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/setup.py

@@ -3,11 +3,9 @@ from setuptools import setup
 
 # requirements
 install_requires = [
-    "authlib",  # FIXME: drop after next release of confluent-kafka with OIDC support
-    "confluent-kafka >= 1.6.1, != 2.1.0, != 2.1.1",
+    "confluent-kafka >= 2.11.0",
     "dataclasses ; python_version < '3.7'",
     "importlib-metadata ; python_version < '3.8'",
-    "requests",  # FIXME: drop after next release of confluent-kafka with OIDC support
     "tqdm",
     "certifi>=2020.04.05.1",
     "typing-extensions ; python_version < '3.8'",

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/tests/test_auth.py

@@ -1,6 +1,6 @@
 import pytest
 
-from adc.auth import SASLAuth, SASLMethod
+from adc.auth import SASLAuth
 
 
 @pytest.mark.parametrize('auth,expected_config', [

{adc-streaming-2.4.0 → adc_streaming-2.6.0}/tests/test_kafka_integration.py

@@ -2,8 +2,8 @@ import logging
 import tempfile
 import time
 import unittest
-from datetime import datetime, timedelta, timezone
-from typing import List
+from datetime import datetime, timedelta
+from typing import List, Optional
 
 import docker
 import pytest
@@ -59,6 +59,69 @@ class KafkaIntegrationTestCase(unittest.TestCase):
         self.assertEqual(msg.topic(), topic)
         self.assertEqual(msg.value(), b"can you hear me?")
 
+    def test_message_with_key(self):
+        """Try writing a message into the Kafka broker, and try pulling the same
+        message back out.
+
+        """
+        topic = "test_message_with_key"
+        # Push one message in...
+        simple_write_msg(self.kafka, topic, "can you hear me?", key="test_msg")
+        # ... and pull it back out.
+        consumer = adc.consumer.Consumer(adc.consumer.ConsumerConfig(
+            broker_urls=[self.kafka.address],
+            group_id="test_consumer",
+            auth=self.kafka.auth,
+        ))
+        consumer.subscribe(topic)
+        stream = consumer.stream()
+
+        msg = next(stream)
+        if msg.error() is not None:
+            raise Exception(msg.error())
+
+        self.assertEqual(msg.topic(), topic)
+        self.assertEqual(msg.value(), b"can you hear me?")
+        self.assertEqual(msg.key(), b"test_msg")
+
+    def test_message_with_callback(self):
+        """Try writing a message into the Kafka broker, with a delivery callback.
+        """
+        topic = "test_message_with_callback"
+        callback_called = False
+        callback_error = False
+
+        def callback(err, msg):
+            nonlocal callback_called
+            nonlocal callback_error
+            callback_called = True
+            callback_error = err is not None
+
+        ap = adc.producer
+        with ap.Producer(ap.ProducerConfig(broker_urls=[self.kafka.address],
+                                           topic=topic, auth=self.kafka.auth)) as producer:
+            producer.write("message data", delivery_callback=callback)
+            producer.flush()
+            consumer = adc.consumer.Consumer(adc.consumer.ConsumerConfig(
+                broker_urls=[self.kafka.address],
+                group_id="test_consumer",
+                auth=self.kafka.auth,
+            ))
+            consumer.subscribe(topic)
+            stream = consumer.stream()
+
+            msg = next(stream)
+            if msg.error() is not None:
+                raise Exception(msg.error())
+            # give the producer's background thread time to notice the ack from the broker and
+            # fire the callback
+            producer._producer.poll(0.1)
+
+        self.assertEqual(msg.topic(), topic)
+        self.assertEqual(msg.value(), b"message data")
+        self.assertEqual(callback_called, True)
+        self.assertEqual(callback_error, False)
+
     def test_reset_to_end(self):
         # Write a few messages.
         topic = "test_reset_to_end"
@@ -283,7 +346,7 @@ class KafkaIntegrationTestCase(unittest.TestCase):
             "message 1",
             "message 2",
             "message 3",
-        ])        
+        ])
         # Wait a while, write, and wait some more
         time.sleep(2)
         client_middle_time = datetime.now()
@@ -390,8 +453,8 @@ class KafkaIntegrationTestCase(unittest.TestCase):
             topic=None,
             auth=self.kafka.auth,
         ))
-        for i in range(0,8):
-            producer.write(str(i), topic=topics[i%2])
+        for i in range(0, 8):
+            producer.write(str(i), topic=topics[i % 2])
         producer.flush()
         logger.info("messages sent")
 
@@ -403,12 +466,12 @@ class KafkaIntegrationTestCase(unittest.TestCase):
         ))
         consumer.subscribe(topics)
         stream = consumer.stream()
-        total_messages = 0;
+        total_messages = 0
         for msg in stream:
             if msg.error() is not None:
                 raise Exception(msg.error())
             idx = int(msg.value())
-            self.assertEqual(msg.topic(), topics[idx%2])
+            self.assertEqual(msg.topic(), topics[idx % 2])
             total_messages += 1
             if total_messages == 8:
                 break
@@ -451,13 +514,17 @@ class KafkaDockerConnection:
         self.auth = adc.auth.SASLAuth(
             user="test", password="test-pass",
             ssl_ca_location=self.certfile.name,
+            # disable endpoint verification because the docker service generates a certificate with
+            # a useless subject (the container ID) which can never match the hostname used to
+            # connect (typically 0.0.0.0)
+            ssl_endpoint_identification_algorithm="none",
         )
 
     def poll_for_kafka_broker_address(self, maxiter=20, sleep=timedelta(milliseconds=500)):
         """Block until the Docker daemon tells us the IP and Port of the Kafa broker.
 
         Returns the ip and port as a string in the form "ip:port."
-"""
+        """
         i = 0
         while (not self.query_kafka_broker_address()) and i < maxiter:
             logger.info("polling to wait for container to acquire port...")
@@ -479,7 +546,7 @@ class KafkaDockerConnection:
         port = addrs[0]['HostPort']
         return f"{ip}:{port}"
 
-    def poll_for_kafka_active(self, maxiter=20, sleep=timedelta(milliseconds=500)):
+    def poll_for_kafka_active(self, maxiter=40, sleep=timedelta(milliseconds=500)):
         """Block until Kafka's network listener is accepting connections."""
         i = 0
         while (not self.query_kafka_active()) and i < maxiter:
@@ -550,7 +617,7 @@ class KafkaDockerConnection:
             # these tests cannot run if there is already an instance of Kafka running on
             # the same host.
             ports={"9092/tcp": 9092},
-            command=["/root/runServer","--advertisedListener","SASL_SSL://localhost:9092"],
+            command=["/root/runServer", "--advertisedListener", "SASL_SSL://localhost:9092"],
         )
 
     def get_or_create_docker_network(self):
@@ -565,13 +632,13 @@ class KafkaDockerConnection:
         return self.docker_client.networks.create(name="adc-integration-test")
 
 
-def simple_write_msg(conn: KafkaDockerConnection, topic: str, msg: str):
+def simple_write_msg(conn: KafkaDockerConnection, topic: str, msg: str, key: Optional[str] = None):
     producer = adc.producer.Producer(adc.producer.ProducerConfig(
         broker_urls=[conn.address],
         topic=topic,
         auth=conn.auth,
     ))
-    producer.write(msg)
+    producer.write(msg, key=key)
     producer.flush()
 
 

adc-streaming-2.4.0/adc/oidc.py

@@ -1,27 +0,0 @@
-def set_oauth_cb(config):
-    """Implement client support for KIP-768 OpenID Connect.
-
-    Apache Kafka 3.1.0 supports authentication using OpenID Client Credentials.
-    Native support for Python is still incomplete due to this issue:
-    https://github.com/confluentinc/librdkafka/issues/3751
-
-    Meanwhile, this is a pure Python implementation of the refresh token
-    callback.
-    """
-    if config.pop('sasl.oauthbearer.method', None) != 'oidc':
-        return
-
-    client_id = config.pop('sasl.oauthbearer.client.id')
-    client_secret = config.pop('sasl.oauthbearer.client.secret')
-    scope = config.pop('sasl.oauthbearer.scope', None)
-    token_endpoint = config.pop('sasl.oauthbearer.token.endpoint.url')
-
-    from authlib.integrations.requests_client import OAuth2Session
-    session = OAuth2Session(client_id, client_secret, scope=scope)
-
-    def oauth_cb(*_, **__):
-        token = session.fetch_token(
-            token_endpoint, grant_type='client_credentials')
-        return token['access_token'], token['expires_at']
-
-    config['oauth_cb'] = oauth_cb