adc-streaming 2.5.0__tar.gz → 2.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/.github/workflows/build.yml +1 -1
  2. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/Makefile +1 -1
  3. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/PKG-INFO +30 -3
  4. adc_streaming-2.7.0/adc/auth.py +147 -0
  5. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/consumer.py +0 -2
  6. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/kafka.py +2 -2
  7. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/producer.py +21 -8
  8. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/PKG-INFO +30 -3
  9. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/SOURCES.txt +2 -1
  10. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/requires.txt +1 -3
  11. adc_streaming-2.7.0/adc_streaming.egg-info/scm_file_list.json +32 -0
  12. adc_streaming-2.7.0/adc_streaming.egg-info/scm_version.json +8 -0
  13. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/setup.py +1 -3
  14. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/tests/test_auth.py +10 -1
  15. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/tests/test_kafka_integration.py +46 -3
  16. adc-streaming-2.5.0/adc/auth.py +0 -86
  17. adc-streaming-2.5.0/adc/oidc.py +0 -27
  18. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/.gitignore +0 -0
  19. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/LICENSE +0 -0
  20. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/README.md +0 -0
  21. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/__init__.py +0 -0
  22. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/errors.py +0 -0
  23. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/io.py +0 -0
  24. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/dependency_links.txt +0 -0
  25. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/not-zip-safe +0 -0
  26. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/top_level.txt +0 -0
  27. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/Makefile +0 -0
  28. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/_static/css/my_theme.css +0 -0
  29. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/_templates/layout.html +0 -0
  30. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/api/api.rst +0 -0
  31. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/api/streaming.rst +0 -0
  32. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/conf.py +0 -0
  33. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/index.rst +0 -0
  34. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/user/installation.rst +0 -0
  35. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/user/quickstart.rst +0 -0
  36. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/pyproject.toml +0 -0
  37. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/recipe/meta.yaml +0 -0
  38. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/setup.cfg +0 -0
  39. {adc-streaming-2.5.0 → adc_streaming-2.7.0}/tests/test_kafka.py +0 -0
@@ -7,7 +7,7 @@ jobs:
7
7
  runs-on: ubuntu-latest
8
8
  strategy:
9
9
  matrix:
10
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
10
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
11
11
 
12
12
  steps:
13
13
  - name: Check out the code
@@ -32,7 +32,7 @@ lint :
32
32
  # stop the build if there are Python syntax errors or undefined names
33
33
  flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
34
34
  # exit-zero treats all errors as warnings
35
- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics
35
+ flake8 . --count --exit-zero --max-complexity=20 --max-line-length=100 --statistics
36
36
 
37
37
  .PHONY: format
38
38
  format :
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: adc-streaming
3
- Version: 2.5.0
3
+ Version: 2.7.0
4
4
  Summary: Astronomy Data Commons streaming client libraries
5
5
  Home-page: https://github.com/astronomy-commons/adc-streaming
6
6
  Author: Astronomy Data Commons Team
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
12
12
  Classifier: Operating System :: POSIX :: Linux
13
13
  Classifier: Operating System :: MacOS :: MacOS X
14
14
  Description-Content-Type: text/markdown
15
- Provides-Extra: dev
16
15
  License-File: LICENSE
16
+ Requires-Dist: confluent-kafka>=2.11.0
17
+ Requires-Dist: dataclasses; python_version < "3.7"
18
+ Requires-Dist: importlib-metadata; python_version < "3.8"
19
+ Requires-Dist: tqdm
20
+ Requires-Dist: certifi>=2020.04.05.1
21
+ Requires-Dist: typing-extensions; python_version < "3.8"
22
+ Provides-Extra: dev
23
+ Requires-Dist: autopep8; extra == "dev"
24
+ Requires-Dist: docker; extra == "dev"
25
+ Requires-Dist: flake8; extra == "dev"
26
+ Requires-Dist: isort; extra == "dev"
27
+ Requires-Dist: pytest; extra == "dev"
28
+ Requires-Dist: pytest-timeout; extra == "dev"
29
+ Requires-Dist: pytest-integration; extra == "dev"
30
+ Requires-Dist: sphinx; extra == "dev"
31
+ Requires-Dist: sphinx_rtd_theme; extra == "dev"
32
+ Requires-Dist: twine; extra == "dev"
33
+ Dynamic: author
34
+ Dynamic: author-email
35
+ Dynamic: classifier
36
+ Dynamic: description
37
+ Dynamic: description-content-type
38
+ Dynamic: home-page
39
+ Dynamic: license
40
+ Dynamic: license-file
41
+ Dynamic: provides-extra
42
+ Dynamic: requires-dist
43
+ Dynamic: summary
17
44
 
18
45
  # Astronomy Data Commons Streaming Client Libraries
19
46
 
@@ -0,0 +1,147 @@
1
+ #!/usr/bin/env python
2
+
3
+ import base64
4
+ from collections.abc import Mapping
5
+ from enum import Enum
6
+ import json
7
+ import logging
8
+ import subprocess
9
+
10
+ import certifi
11
+
12
+ logger = logging.getLogger("adc-streaming.auth")
13
+
14
+
15
+ class SASLMethod(Enum):
16
+ """SASL method to use for authentication.
17
+ """
18
+
19
+ PLAIN = 1
20
+ SCRAM_SHA_256 = 2
21
+ SCRAM_SHA_512 = 3
22
+ OAUTHBEARER = 4
23
+
24
+ def __str__(self):
25
+ return self.name.replace("_", "-")
26
+
27
+
28
+ class SASLAuth(object):
29
+ """Attach SASL-based authentication to a client.
30
+
31
+ Returns client-based auth options when called.
32
+
33
+ Parameters
34
+ ----------
35
+ user : `str`
36
+ Username to authenticate with.
37
+ password : `str`
38
+ Password to authenticate with.
39
+ ssl : `bool`, optional
40
+ Whether to enable SSL (enabled by default).
41
+ method : `SASLMethod`, optional
42
+ The SASL method to authenticate. The default is SASLMethod.OAUTHBEARER
43
+ if token_endpoint is provided, or SASLMethod.PLAIN otherwise.
44
+ See valid SASL methods in SASLMethod.
45
+ ssl_ca_location : `str`, optional
46
+ If using SSL via a self-signed cert, a path/location
47
+ to the certificate.
48
+ ssl_endpoint_identification_algorithm : `str`, optional
49
+ If using SSL, the algorithm used to verify that certificate is valid for the endpoint.
50
+ token_endpoint : `str`, optional
51
+ The OpenID Connect token endpoint URL.
52
+ Required for OAUTHBEARER / OpenID Connect, otherwise ignored.
53
+
54
+ """
55
+
56
+ def __init__(self, user, password, ssl=True, method=None, token_endpoint=None, **kwargs):
57
+ if method is None:
58
+ if token_endpoint is not None or "token_command" in kwargs:
59
+ method = SASLMethod.OAUTHBEARER
60
+ else:
61
+ method = SASLMethod.PLAIN
62
+
63
+ self._method = method
64
+
65
+ # set up SSL options
66
+ if ssl:
67
+ if "ssl_ca_location" in kwargs:
68
+ ssl_cert = kwargs["ssl_ca_location"]
69
+ else:
70
+ ssl_cert = certifi.where()
71
+
72
+ self._config = {
73
+ "security.protocol": "SASL_SSL",
74
+ "ssl.ca.location": ssl_cert,
75
+ "https.ca.location": ssl_cert,
76
+ }
77
+ if "ssl_endpoint_identification_algorithm" in kwargs:
78
+ self._config["ssl.endpoint.identification.algorithm"] = \
79
+ kwargs["ssl_endpoint_identification_algorithm"]
80
+ else:
81
+ self._config = {"security.protocol": "SASL_PLAINTEXT"}
82
+
83
+ # set up SASL options
84
+ self._config["sasl.mechanism"] = str(self._method)
85
+ if method == SASLMethod.OAUTHBEARER:
86
+ if token_endpoint is not None:
87
+ self._config["sasl.oauthbearer.client.id"] = user
88
+ self._config["sasl.oauthbearer.client.secret"] = password
89
+ self._config["sasl.oauthbearer.method"] = "oidc"
90
+ self._config["sasl.oauthbearer.token.endpoint.url"] = token_endpoint
91
+ elif "token_command" in kwargs:
92
+ token_command = kwargs["token_command"]
93
+ self._config["sasl.oauthbearer.method"] = "default"
94
+ self._config["oauth_cb"] = lambda c: SASLAuth.external_token_callback(token_command)
95
+ else:
96
+ self._config["sasl.username"] = user
97
+ self._config["sasl.password"] = password
98
+
99
+ @staticmethod
100
+ def external_token_callback(command):
101
+ try:
102
+ presult = subprocess.run(command, shell=True, capture_output=True)
103
+ if presult.returncode != 0:
104
+ raise RuntimeError("Token callback command failed: " + presult.stderr)
105
+ rawdata = presult.stdout
106
+ # need to parse the resulting JWT enough to extract the expiration time
107
+ # we do no other validation, since downstream systems should be responsible
108
+ # for that already
109
+ sections = rawdata.split(b'.')
110
+ if len(sections) != 3:
111
+ raise RuntimeError("Token callback output does not appear to be a valid JWT")
112
+ try:
113
+ # The JWT spec mandates that base 64 padding be omitted, but
114
+ # base64.urlsafe_b64decode requires it, so we must put it back.
115
+ m = len(sections[1]) % 4
116
+ if m == 2:
117
+ sections[1] += b"=="
118
+ elif m == 3:
119
+ sections[1] += b"="
120
+ elif m == 1:
121
+ raise RuntimeError("Token callback output is not valid base64 data")
122
+ claims = json.loads(base64.urlsafe_b64decode(sections[1]).decode("utf-8"))
123
+ except UnicodeDecodeError:
124
+ raise RuntimeError("Token callback output is not valid UTF-8 after base64 decoding")
125
+ except json.JSONDecodeError:
126
+ raise RuntimeError("Token callback output is not valid JSON after base64 and "
127
+ "UTF-8 decoding")
128
+ if not isinstance(claims, Mapping):
129
+ raise RuntimeError("Token callback output does not contain valid claims")
130
+ if "sub" not in claims:
131
+ raise RuntimeError("Token callback output does not contain an subject claim")
132
+ if "exp" not in claims:
133
+ raise RuntimeError("Token callback output does not contain an expiration claim")
134
+ exp_value = claims["exp"]
135
+ if not isinstance(exp_value, int) and not isinstance(exp_value, float):
136
+ raise RuntimeError("Token expiration value is not a number")
137
+ exp_value = float(exp_value)
138
+ try:
139
+ return (rawdata.decode("utf-8").strip(), exp_value, claims["sub"], {})
140
+ except UnicodeDecodeError:
141
+ raise RuntimeError("Token callback output is not valid UTF-8 data")
142
+ except Exception as ex:
143
+ logger.error(f"Token callback error: {ex}")
144
+ raise
145
+
146
+ def __call__(self):
147
+ return self._config
@@ -14,7 +14,6 @@ import confluent_kafka.admin # type: ignore
14
14
 
15
15
  from .auth import SASLAuth
16
16
  from .errors import ErrorCallback, log_client_errors
17
- from .oidc import set_oauth_cb
18
17
 
19
18
 
20
19
  class LogicalOffset(enum.IntEnum):
@@ -360,5 +359,4 @@ class ConsumerConfig:
360
359
 
361
360
  if self.auth is not None:
362
361
  config.update(self.auth())
363
- set_oauth_cb(config)
364
362
  return config
@@ -14,12 +14,12 @@ def parse_kafka_url(val):
14
14
  if parsed.scheme != "kafka":
15
15
  raise ValueError("invalid kafka URL: must start with 'kafka://'")
16
16
 
17
+ group_id = parsed.username
18
+ # intentionally ignore passwords embedded in URLs to discourage their use
17
19
  split_netloc = parsed.netloc.split("@", maxsplit=1)
18
20
  if len(split_netloc) == 2:
19
- group_id = split_netloc[0]
20
21
  broker_addresses = split_netloc[1].split(",")
21
22
  else:
22
- group_id = None
23
23
  broker_addresses = split_netloc[0].split(",")
24
24
 
25
25
  topics = parsed.path.lstrip("/")
@@ -14,7 +14,6 @@ import confluent_kafka # type: ignore
14
14
  from .auth import SASLAuth
15
15
  from .errors import (DeliveryCallback, ErrorCallback, log_client_errors,
16
16
  log_delivery_errors)
17
- from .oidc import set_oauth_cb
18
17
 
19
18
 
20
19
  class Producer:
@@ -48,11 +47,26 @@ class Producer:
48
47
  "Either configure a topic when constructing the Producer, "
49
48
  "or specify the topic argument to write()")
50
49
  self.logger.debug("writing message to %s", topic)
50
+ produce_kwargs = {"headers": headers, "key": key}
51
51
  if delivery_callback is not None:
52
- self._producer.produce(topic, msg, headers=headers, key=key,
53
- on_delivery=delivery_callback)
54
- else:
55
- self._producer.produce(topic, msg, headers=headers, key=key,)
52
+ produce_kwargs["on_delivery"] = delivery_callback
53
+ while True:
54
+ try:
55
+ self._producer.produce(topic, msg, **produce_kwargs)
56
+ break
57
+ except BufferError:
58
+ # It's hard to know what the size limit on the buffer is, so we will try to
59
+ # wait until the number of items in it decreases (or it is empty, in case all
60
+ # messages were successfully sent in the time it took to handle the error).
61
+ buffer_len = len(self._producer)
62
+ self.logger.debug(f"Blocking due to BufferError, buffer size: {buffer_len}")
63
+ while buffer_len > 0 and len(self._producer) == buffer_len:
64
+ self._producer.poll(0.01)
65
+
66
+ def queued_message_count(self):
67
+ """Get the number of messages waiting to be sent to the broker.
68
+ """
69
+ return len(self._producer)
56
70
 
57
71
  def flush(self, timeout: timedelta = timedelta(seconds=10)) -> int:
58
72
  """Attempt to flush enqueued messages. Return the number of messages still
@@ -66,9 +80,9 @@ class Producer:
66
80
  self.logger.debug("flushed all messages")
67
81
  return n
68
82
 
69
- def close(self) -> int:
83
+ def close(self, timeout: timedelta = timedelta(seconds=10)) -> int:
70
84
  self.logger.debug("shutting down producer")
71
- return self.flush()
85
+ return self.flush(timeout)
72
86
 
73
87
  def __enter__(self) -> 'Producer':
74
88
  return self
@@ -142,7 +156,6 @@ class ProducerConfig:
142
156
  config["error_cb"] = self.error_callback
143
157
  if self.auth is not None:
144
158
  config.update(self.auth())
145
- set_oauth_cb(config)
146
159
  return config
147
160
 
148
161
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: adc-streaming
3
- Version: 2.5.0
3
+ Version: 2.7.0
4
4
  Summary: Astronomy Data Commons streaming client libraries
5
5
  Home-page: https://github.com/astronomy-commons/adc-streaming
6
6
  Author: Astronomy Data Commons Team
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
12
12
  Classifier: Operating System :: POSIX :: Linux
13
13
  Classifier: Operating System :: MacOS :: MacOS X
14
14
  Description-Content-Type: text/markdown
15
- Provides-Extra: dev
16
15
  License-File: LICENSE
16
+ Requires-Dist: confluent-kafka>=2.11.0
17
+ Requires-Dist: dataclasses; python_version < "3.7"
18
+ Requires-Dist: importlib-metadata; python_version < "3.8"
19
+ Requires-Dist: tqdm
20
+ Requires-Dist: certifi>=2020.04.05.1
21
+ Requires-Dist: typing-extensions; python_version < "3.8"
22
+ Provides-Extra: dev
23
+ Requires-Dist: autopep8; extra == "dev"
24
+ Requires-Dist: docker; extra == "dev"
25
+ Requires-Dist: flake8; extra == "dev"
26
+ Requires-Dist: isort; extra == "dev"
27
+ Requires-Dist: pytest; extra == "dev"
28
+ Requires-Dist: pytest-timeout; extra == "dev"
29
+ Requires-Dist: pytest-integration; extra == "dev"
30
+ Requires-Dist: sphinx; extra == "dev"
31
+ Requires-Dist: sphinx_rtd_theme; extra == "dev"
32
+ Requires-Dist: twine; extra == "dev"
33
+ Dynamic: author
34
+ Dynamic: author-email
35
+ Dynamic: classifier
36
+ Dynamic: description
37
+ Dynamic: description-content-type
38
+ Dynamic: home-page
39
+ Dynamic: license
40
+ Dynamic: license-file
41
+ Dynamic: provides-extra
42
+ Dynamic: requires-dist
43
+ Dynamic: summary
17
44
 
18
45
  # Astronomy Data Commons Streaming Client Libraries
19
46
 
@@ -12,13 +12,14 @@ adc/consumer.py
12
12
  adc/errors.py
13
13
  adc/io.py
14
14
  adc/kafka.py
15
- adc/oidc.py
16
15
  adc/producer.py
17
16
  adc_streaming.egg-info/PKG-INFO
18
17
  adc_streaming.egg-info/SOURCES.txt
19
18
  adc_streaming.egg-info/dependency_links.txt
20
19
  adc_streaming.egg-info/not-zip-safe
21
20
  adc_streaming.egg-info/requires.txt
21
+ adc_streaming.egg-info/scm_file_list.json
22
+ adc_streaming.egg-info/scm_version.json
22
23
  adc_streaming.egg-info/top_level.txt
23
24
  doc/Makefile
24
25
  doc/conf.py
@@ -1,6 +1,4 @@
1
- authlib
2
- confluent-kafka!=2.1.0,!=2.1.1,>=1.6.1
3
- requests
1
+ confluent-kafka>=2.11.0
4
2
  tqdm
5
3
  certifi>=2020.04.05.1
6
4
 
@@ -0,0 +1,32 @@
1
+ {
2
+ "files": [
3
+ ".github/workflows/build.yml",
4
+ ".gitignore",
5
+ "LICENSE",
6
+ "Makefile",
7
+ "README.md",
8
+ "adc/__init__.py",
9
+ "adc/auth.py",
10
+ "adc/consumer.py",
11
+ "adc/errors.py",
12
+ "adc/io.py",
13
+ "adc/kafka.py",
14
+ "adc/producer.py",
15
+ "doc/Makefile",
16
+ "doc/_static/css/my_theme.css",
17
+ "doc/_templates/layout.html",
18
+ "doc/api/api.rst",
19
+ "doc/api/streaming.rst",
20
+ "doc/conf.py",
21
+ "doc/index.rst",
22
+ "doc/user/installation.rst",
23
+ "doc/user/quickstart.rst",
24
+ "pyproject.toml",
25
+ "recipe/meta.yaml",
26
+ "setup.cfg",
27
+ "setup.py",
28
+ "tests/test_auth.py",
29
+ "tests/test_kafka.py",
30
+ "tests/test_kafka_integration.py"
31
+ ]
32
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "2.6.0",
3
+ "distance": 5,
4
+ "node": "ge0095abe7c2bd5a1dce1a913653bdc084fcd2496",
5
+ "dirty": false,
6
+ "branch": "external-token-support",
7
+ "node_date": "2026-08-28"
8
+ }
@@ -3,11 +3,9 @@ from setuptools import setup
3
3
 
4
4
  # requirements
5
5
  install_requires = [
6
- "authlib", # FIXME: drop after next release of confluent-kafka with OIDC support
7
- "confluent-kafka >= 1.6.1, != 2.1.0, != 2.1.1",
6
+ "confluent-kafka >= 2.11.0",
8
7
  "dataclasses ; python_version < '3.7'",
9
8
  "importlib-metadata ; python_version < '3.8'",
10
- "requests", # FIXME: drop after next release of confluent-kafka with OIDC support
11
9
  "tqdm",
12
10
  "certifi>=2020.04.05.1",
13
11
  "typing-extensions ; python_version < '3.8'",
@@ -28,7 +28,16 @@ from adc.auth import SASLAuth
28
28
  'sasl.password': 'test-pass',
29
29
  'security.protocol': 'SASL_SSL'
30
30
  }
31
- ]
31
+ ],
32
+ [
33
+ SASLAuth(
34
+ 'test', '', token_command="generate_token"
35
+ ),
36
+ {
37
+ 'sasl.mechanism': 'OAUTHBEARER',
38
+ 'sasl.oauthbearer.method': 'default',
39
+ }
40
+ ],
32
41
  ])
33
42
  def test_auth(auth, expected_config):
34
43
  # Check that the key/value pairs in expected_config are a subset
@@ -5,7 +5,11 @@ import unittest
5
5
  from datetime import datetime, timedelta
6
6
  from typing import List, Optional
7
7
 
8
- import docker
8
+ try:
9
+ import docker
10
+ have_docker = True
11
+ except ImportError:
12
+ have_docker = False
9
13
  import pytest
10
14
 
11
15
  import adc.consumer
@@ -17,6 +21,7 @@ logging.getLogger("adc-streaming").setLevel(logging.DEBUG)
17
21
  logger = logging.getLogger("adc-streaming.tests")
18
22
 
19
23
 
24
+ @pytest.mark.skipif(not have_docker, reason="requires docker")
20
25
  @pytest.mark.integration_test
21
26
  class KafkaIntegrationTestCase(unittest.TestCase):
22
27
  """This test runs a Kafka broker in a Docker container, and makes sure that
@@ -84,6 +89,44 @@ class KafkaIntegrationTestCase(unittest.TestCase):
84
89
  self.assertEqual(msg.value(), b"can you hear me?")
85
90
  self.assertEqual(msg.key(), b"test_msg")
86
91
 
92
+ def test_message_with_callback(self):
93
+ """Try writing a message into the Kafka broker, with a delivery callback.
94
+ """
95
+ topic = "test_message_with_callback"
96
+ callback_called = False
97
+ callback_error = False
98
+
99
+ def callback(err, msg):
100
+ nonlocal callback_called
101
+ nonlocal callback_error
102
+ callback_called = True
103
+ callback_error = err is not None
104
+
105
+ ap = adc.producer
106
+ with ap.Producer(ap.ProducerConfig(broker_urls=[self.kafka.address],
107
+ topic=topic, auth=self.kafka.auth)) as producer:
108
+ producer.write("message data", delivery_callback=callback)
109
+ producer.flush()
110
+ consumer = adc.consumer.Consumer(adc.consumer.ConsumerConfig(
111
+ broker_urls=[self.kafka.address],
112
+ group_id="test_consumer",
113
+ auth=self.kafka.auth,
114
+ ))
115
+ consumer.subscribe(topic)
116
+ stream = consumer.stream()
117
+
118
+ msg = next(stream)
119
+ if msg.error() is not None:
120
+ raise Exception(msg.error())
121
+ # give the producer's background thread time to notice the ack from the broker and
122
+ # fire the callback
123
+ producer._producer.poll(0.1)
124
+
125
+ self.assertEqual(msg.topic(), topic)
126
+ self.assertEqual(msg.value(), b"message data")
127
+ self.assertEqual(callback_called, True)
128
+ self.assertEqual(callback_error, False)
129
+
87
130
  def test_reset_to_end(self):
88
131
  # Write a few messages.
89
132
  topic = "test_reset_to_end"
@@ -486,7 +529,7 @@ class KafkaDockerConnection:
486
529
  """Block until the Docker daemon tells us the IP and Port of the Kafa broker.
487
530
 
488
531
  Returns the ip and port as a string in the form "ip:port."
489
- """
532
+ """
490
533
  i = 0
491
534
  while (not self.query_kafka_broker_address()) and i < maxiter:
492
535
  logger.info("polling to wait for container to acquire port...")
@@ -508,7 +551,7 @@ class KafkaDockerConnection:
508
551
  port = addrs[0]['HostPort']
509
552
  return f"{ip}:{port}"
510
553
 
511
- def poll_for_kafka_active(self, maxiter=20, sleep=timedelta(milliseconds=500)):
554
+ def poll_for_kafka_active(self, maxiter=40, sleep=timedelta(milliseconds=500)):
512
555
  """Block until Kafka's network listener is accepting connections."""
513
556
  i = 0
514
557
  while (not self.query_kafka_active()) and i < maxiter:
@@ -1,86 +0,0 @@
1
- #!/usr/bin/env python
2
-
3
- from enum import Enum
4
-
5
- import certifi
6
-
7
-
8
- class SASLMethod(Enum):
9
- """SASL method to use for authentication.
10
- """
11
-
12
- PLAIN = 1
13
- SCRAM_SHA_256 = 2
14
- SCRAM_SHA_512 = 3
15
- OAUTHBEARER = 4
16
-
17
- def __str__(self):
18
- return self.name.replace("_", "-")
19
-
20
-
21
- class SASLAuth(object):
22
- """Attach SASL-based authentication to a client.
23
-
24
- Returns client-based auth options when called.
25
-
26
- Parameters
27
- ----------
28
- user : `str`
29
- Username to authenticate with.
30
- password : `str`
31
- Password to authenticate with.
32
- ssl : `bool`, optional
33
- Whether to enable SSL (enabled by default).
34
- method : `SASLMethod`, optional
35
- The SASL method to authenticate. The default is SASLMethod.OAUTHBEARER
36
- if token_endpoint is provided, or SASLMethod.PLAIN otherwise.
37
- See valid SASL methods in SASLMethod.
38
- ssl_ca_location : `str`, optional
39
- If using SSL via a self-signed cert, a path/location
40
- to the certificate.
41
- ssl_endpoint_identification_algorithm : `str`, optional
42
- If using SSL, the algorithm used to verify that certificate is valid for the endpoint.
43
- token_endpoint : `str`, optional
44
- The OpenID Connect token endpoint URL.
45
- Required for OAUTHBEARER / OpenID Connect, otherwise ignored.
46
-
47
- """
48
-
49
- def __init__(self, user, password, ssl=True, method=None, token_endpoint=None, **kwargs):
50
- if method is None:
51
- if token_endpoint is None:
52
- method = SASLMethod.PLAIN
53
- else:
54
- method = SASLMethod.OAUTHBEARER
55
- self._method = method
56
-
57
- # set up SSL options
58
- if ssl:
59
- if "ssl_ca_location" in kwargs:
60
- ssl_cert = kwargs["ssl_ca_location"]
61
- else:
62
- ssl_cert = certifi.where()
63
-
64
- self._config = {
65
- "security.protocol": "SASL_SSL",
66
- "ssl.ca.location": ssl_cert,
67
- }
68
- if "ssl_endpoint_identification_algorithm" in kwargs:
69
- self._config["ssl.endpoint.identification.algorithm"] = \
70
- kwargs["ssl_endpoint_identification_algorithm"]
71
- else:
72
- self._config = {"security.protocol": "SASL_PLAINTEXT"}
73
-
74
- # set up SASL options
75
- self._config["sasl.mechanism"] = str(self._method)
76
- if token_endpoint:
77
- self._config["sasl.oauthbearer.client.id"] = user
78
- self._config["sasl.oauthbearer.client.secret"] = password
79
- self._config["sasl.oauthbearer.method"] = "oidc"
80
- self._config["sasl.oauthbearer.token.endpoint.url"] = token_endpoint
81
- else:
82
- self._config["sasl.username"] = user
83
- self._config["sasl.password"] = password
84
-
85
- def __call__(self):
86
- return self._config
@@ -1,27 +0,0 @@
1
- def set_oauth_cb(config):
2
- """Implement client support for KIP-768 OpenID Connect.
3
-
4
- Apache Kafka 3.1.0 supports authentication using OpenID Client Credentials.
5
- Native support for Python is still incomplete due to this issue:
6
- https://github.com/confluentinc/librdkafka/issues/3751
7
-
8
- Meanwhile, this is a pure Python implementation of the refresh token
9
- callback.
10
- """
11
- if config.pop('sasl.oauthbearer.method', None) != 'oidc':
12
- return
13
-
14
- client_id = config.pop('sasl.oauthbearer.client.id')
15
- client_secret = config.pop('sasl.oauthbearer.client.secret')
16
- scope = config.pop('sasl.oauthbearer.scope', None)
17
- token_endpoint = config.pop('sasl.oauthbearer.token.endpoint.url')
18
-
19
- from authlib.integrations.requests_client import OAuth2Session
20
- session = OAuth2Session(client_id, client_secret, scope=scope)
21
-
22
- def oauth_cb(*_, **__):
23
- token = session.fetch_token(
24
- token_endpoint, grant_type='client_credentials')
25
- return token['access_token'], token['expires_at']
26
-
27
- config['oauth_cb'] = oauth_cb
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes