adc-streaming 2.5.0__tar.gz → 2.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/.github/workflows/build.yml +1 -1
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/Makefile +1 -1
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/PKG-INFO +30 -3
- adc_streaming-2.7.0/adc/auth.py +147 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/consumer.py +0 -2
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/kafka.py +2 -2
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/producer.py +21 -8
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/PKG-INFO +30 -3
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/SOURCES.txt +2 -1
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/requires.txt +1 -3
- adc_streaming-2.7.0/adc_streaming.egg-info/scm_file_list.json +32 -0
- adc_streaming-2.7.0/adc_streaming.egg-info/scm_version.json +8 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/setup.py +1 -3
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/tests/test_auth.py +10 -1
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/tests/test_kafka_integration.py +46 -3
- adc-streaming-2.5.0/adc/auth.py +0 -86
- adc-streaming-2.5.0/adc/oidc.py +0 -27
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/.gitignore +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/LICENSE +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/README.md +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/__init__.py +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/errors.py +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc/io.py +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/dependency_links.txt +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/not-zip-safe +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/adc_streaming.egg-info/top_level.txt +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/Makefile +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/_static/css/my_theme.css +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/_templates/layout.html +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/api/api.rst +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/api/streaming.rst +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/conf.py +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/index.rst +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/user/installation.rst +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/doc/user/quickstart.rst +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/pyproject.toml +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/recipe/meta.yaml +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/setup.cfg +0 -0
- {adc-streaming-2.5.0 → adc_streaming-2.7.0}/tests/test_kafka.py +0 -0
|
@@ -32,7 +32,7 @@ lint :
|
|
|
32
32
|
# stop the build if there are Python syntax errors or undefined names
|
|
33
33
|
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
|
34
34
|
# exit-zero treats all errors as warnings
|
|
35
|
-
flake8 . --count --exit-zero --max-complexity=
|
|
35
|
+
flake8 . --count --exit-zero --max-complexity=20 --max-line-length=100 --statistics
|
|
36
36
|
|
|
37
37
|
.PHONY: format
|
|
38
38
|
format :
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: adc-streaming
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Astronomy Data Commons streaming client libraries
|
|
5
5
|
Home-page: https://github.com/astronomy-commons/adc-streaming
|
|
6
6
|
Author: Astronomy Data Commons Team
|
|
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
|
|
|
12
12
|
Classifier: Operating System :: POSIX :: Linux
|
|
13
13
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
14
14
|
Description-Content-Type: text/markdown
|
|
15
|
-
Provides-Extra: dev
|
|
16
15
|
License-File: LICENSE
|
|
16
|
+
Requires-Dist: confluent-kafka>=2.11.0
|
|
17
|
+
Requires-Dist: dataclasses; python_version < "3.7"
|
|
18
|
+
Requires-Dist: importlib-metadata; python_version < "3.8"
|
|
19
|
+
Requires-Dist: tqdm
|
|
20
|
+
Requires-Dist: certifi>=2020.04.05.1
|
|
21
|
+
Requires-Dist: typing-extensions; python_version < "3.8"
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: autopep8; extra == "dev"
|
|
24
|
+
Requires-Dist: docker; extra == "dev"
|
|
25
|
+
Requires-Dist: flake8; extra == "dev"
|
|
26
|
+
Requires-Dist: isort; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-timeout; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-integration; extra == "dev"
|
|
30
|
+
Requires-Dist: sphinx; extra == "dev"
|
|
31
|
+
Requires-Dist: sphinx_rtd_theme; extra == "dev"
|
|
32
|
+
Requires-Dist: twine; extra == "dev"
|
|
33
|
+
Dynamic: author
|
|
34
|
+
Dynamic: author-email
|
|
35
|
+
Dynamic: classifier
|
|
36
|
+
Dynamic: description
|
|
37
|
+
Dynamic: description-content-type
|
|
38
|
+
Dynamic: home-page
|
|
39
|
+
Dynamic: license
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
Dynamic: provides-extra
|
|
42
|
+
Dynamic: requires-dist
|
|
43
|
+
Dynamic: summary
|
|
17
44
|
|
|
18
45
|
# Astronomy Data Commons Streaming Client Libraries
|
|
19
46
|
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from enum import Enum
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import subprocess
|
|
9
|
+
|
|
10
|
+
import certifi
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger("adc-streaming.auth")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SASLMethod(Enum):
|
|
16
|
+
"""SASL method to use for authentication.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
PLAIN = 1
|
|
20
|
+
SCRAM_SHA_256 = 2
|
|
21
|
+
SCRAM_SHA_512 = 3
|
|
22
|
+
OAUTHBEARER = 4
|
|
23
|
+
|
|
24
|
+
def __str__(self):
|
|
25
|
+
return self.name.replace("_", "-")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class SASLAuth(object):
|
|
29
|
+
"""Attach SASL-based authentication to a client.
|
|
30
|
+
|
|
31
|
+
Returns client-based auth options when called.
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
user : `str`
|
|
36
|
+
Username to authenticate with.
|
|
37
|
+
password : `str`
|
|
38
|
+
Password to authenticate with.
|
|
39
|
+
ssl : `bool`, optional
|
|
40
|
+
Whether to enable SSL (enabled by default).
|
|
41
|
+
method : `SASLMethod`, optional
|
|
42
|
+
The SASL method to authenticate. The default is SASLMethod.OAUTHBEARER
|
|
43
|
+
if token_endpoint is provided, or SASLMethod.PLAIN otherwise.
|
|
44
|
+
See valid SASL methods in SASLMethod.
|
|
45
|
+
ssl_ca_location : `str`, optional
|
|
46
|
+
If using SSL via a self-signed cert, a path/location
|
|
47
|
+
to the certificate.
|
|
48
|
+
ssl_endpoint_identification_algorithm : `str`, optional
|
|
49
|
+
If using SSL, the algorithm used to verify that certificate is valid for the endpoint.
|
|
50
|
+
token_endpoint : `str`, optional
|
|
51
|
+
The OpenID Connect token endpoint URL.
|
|
52
|
+
Required for OAUTHBEARER / OpenID Connect, otherwise ignored.
|
|
53
|
+
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(self, user, password, ssl=True, method=None, token_endpoint=None, **kwargs):
|
|
57
|
+
if method is None:
|
|
58
|
+
if token_endpoint is not None or "token_command" in kwargs:
|
|
59
|
+
method = SASLMethod.OAUTHBEARER
|
|
60
|
+
else:
|
|
61
|
+
method = SASLMethod.PLAIN
|
|
62
|
+
|
|
63
|
+
self._method = method
|
|
64
|
+
|
|
65
|
+
# set up SSL options
|
|
66
|
+
if ssl:
|
|
67
|
+
if "ssl_ca_location" in kwargs:
|
|
68
|
+
ssl_cert = kwargs["ssl_ca_location"]
|
|
69
|
+
else:
|
|
70
|
+
ssl_cert = certifi.where()
|
|
71
|
+
|
|
72
|
+
self._config = {
|
|
73
|
+
"security.protocol": "SASL_SSL",
|
|
74
|
+
"ssl.ca.location": ssl_cert,
|
|
75
|
+
"https.ca.location": ssl_cert,
|
|
76
|
+
}
|
|
77
|
+
if "ssl_endpoint_identification_algorithm" in kwargs:
|
|
78
|
+
self._config["ssl.endpoint.identification.algorithm"] = \
|
|
79
|
+
kwargs["ssl_endpoint_identification_algorithm"]
|
|
80
|
+
else:
|
|
81
|
+
self._config = {"security.protocol": "SASL_PLAINTEXT"}
|
|
82
|
+
|
|
83
|
+
# set up SASL options
|
|
84
|
+
self._config["sasl.mechanism"] = str(self._method)
|
|
85
|
+
if method == SASLMethod.OAUTHBEARER:
|
|
86
|
+
if token_endpoint is not None:
|
|
87
|
+
self._config["sasl.oauthbearer.client.id"] = user
|
|
88
|
+
self._config["sasl.oauthbearer.client.secret"] = password
|
|
89
|
+
self._config["sasl.oauthbearer.method"] = "oidc"
|
|
90
|
+
self._config["sasl.oauthbearer.token.endpoint.url"] = token_endpoint
|
|
91
|
+
elif "token_command" in kwargs:
|
|
92
|
+
token_command = kwargs["token_command"]
|
|
93
|
+
self._config["sasl.oauthbearer.method"] = "default"
|
|
94
|
+
self._config["oauth_cb"] = lambda c: SASLAuth.external_token_callback(token_command)
|
|
95
|
+
else:
|
|
96
|
+
self._config["sasl.username"] = user
|
|
97
|
+
self._config["sasl.password"] = password
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def external_token_callback(command):
|
|
101
|
+
try:
|
|
102
|
+
presult = subprocess.run(command, shell=True, capture_output=True)
|
|
103
|
+
if presult.returncode != 0:
|
|
104
|
+
raise RuntimeError("Token callback command failed: " + presult.stderr)
|
|
105
|
+
rawdata = presult.stdout
|
|
106
|
+
# need to parse the resulting JWT enough to extract the expiration time
|
|
107
|
+
# we do no other validation, since downstream systems should be responsible
|
|
108
|
+
# for that already
|
|
109
|
+
sections = rawdata.split(b'.')
|
|
110
|
+
if len(sections) != 3:
|
|
111
|
+
raise RuntimeError("Token callback output does not appear to be a valid JWT")
|
|
112
|
+
try:
|
|
113
|
+
# The JWT spec mandates that base 64 padding be omitted, but
|
|
114
|
+
# base64.urlsafe_b64decode requires it, so we must put it back.
|
|
115
|
+
m = len(sections[1]) % 4
|
|
116
|
+
if m == 2:
|
|
117
|
+
sections[1] += b"=="
|
|
118
|
+
elif m == 3:
|
|
119
|
+
sections[1] += b"="
|
|
120
|
+
elif m == 1:
|
|
121
|
+
raise RuntimeError("Token callback output is not valid base64 data")
|
|
122
|
+
claims = json.loads(base64.urlsafe_b64decode(sections[1]).decode("utf-8"))
|
|
123
|
+
except UnicodeDecodeError:
|
|
124
|
+
raise RuntimeError("Token callback output is not valid UTF-8 after base64 decoding")
|
|
125
|
+
except json.JSONDecodeError:
|
|
126
|
+
raise RuntimeError("Token callback output is not valid JSON after base64 and "
|
|
127
|
+
"UTF-8 decoding")
|
|
128
|
+
if not isinstance(claims, Mapping):
|
|
129
|
+
raise RuntimeError("Token callback output does not contain valid claims")
|
|
130
|
+
if "sub" not in claims:
|
|
131
|
+
raise RuntimeError("Token callback output does not contain an subject claim")
|
|
132
|
+
if "exp" not in claims:
|
|
133
|
+
raise RuntimeError("Token callback output does not contain an expiration claim")
|
|
134
|
+
exp_value = claims["exp"]
|
|
135
|
+
if not isinstance(exp_value, int) and not isinstance(exp_value, float):
|
|
136
|
+
raise RuntimeError("Token expiration value is not a number")
|
|
137
|
+
exp_value = float(exp_value)
|
|
138
|
+
try:
|
|
139
|
+
return (rawdata.decode("utf-8").strip(), exp_value, claims["sub"], {})
|
|
140
|
+
except UnicodeDecodeError:
|
|
141
|
+
raise RuntimeError("Token callback output is not valid UTF-8 data")
|
|
142
|
+
except Exception as ex:
|
|
143
|
+
logger.error(f"Token callback error: {ex}")
|
|
144
|
+
raise
|
|
145
|
+
|
|
146
|
+
def __call__(self):
|
|
147
|
+
return self._config
|
|
@@ -14,7 +14,6 @@ import confluent_kafka.admin # type: ignore
|
|
|
14
14
|
|
|
15
15
|
from .auth import SASLAuth
|
|
16
16
|
from .errors import ErrorCallback, log_client_errors
|
|
17
|
-
from .oidc import set_oauth_cb
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
class LogicalOffset(enum.IntEnum):
|
|
@@ -360,5 +359,4 @@ class ConsumerConfig:
|
|
|
360
359
|
|
|
361
360
|
if self.auth is not None:
|
|
362
361
|
config.update(self.auth())
|
|
363
|
-
set_oauth_cb(config)
|
|
364
362
|
return config
|
|
@@ -14,12 +14,12 @@ def parse_kafka_url(val):
|
|
|
14
14
|
if parsed.scheme != "kafka":
|
|
15
15
|
raise ValueError("invalid kafka URL: must start with 'kafka://'")
|
|
16
16
|
|
|
17
|
+
group_id = parsed.username
|
|
18
|
+
# intentionally ignore passwords embedded in URLs to discourage their use
|
|
17
19
|
split_netloc = parsed.netloc.split("@", maxsplit=1)
|
|
18
20
|
if len(split_netloc) == 2:
|
|
19
|
-
group_id = split_netloc[0]
|
|
20
21
|
broker_addresses = split_netloc[1].split(",")
|
|
21
22
|
else:
|
|
22
|
-
group_id = None
|
|
23
23
|
broker_addresses = split_netloc[0].split(",")
|
|
24
24
|
|
|
25
25
|
topics = parsed.path.lstrip("/")
|
|
@@ -14,7 +14,6 @@ import confluent_kafka # type: ignore
|
|
|
14
14
|
from .auth import SASLAuth
|
|
15
15
|
from .errors import (DeliveryCallback, ErrorCallback, log_client_errors,
|
|
16
16
|
log_delivery_errors)
|
|
17
|
-
from .oidc import set_oauth_cb
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
class Producer:
|
|
@@ -48,11 +47,26 @@ class Producer:
|
|
|
48
47
|
"Either configure a topic when constructing the Producer, "
|
|
49
48
|
"or specify the topic argument to write()")
|
|
50
49
|
self.logger.debug("writing message to %s", topic)
|
|
50
|
+
produce_kwargs = {"headers": headers, "key": key}
|
|
51
51
|
if delivery_callback is not None:
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
52
|
+
produce_kwargs["on_delivery"] = delivery_callback
|
|
53
|
+
while True:
|
|
54
|
+
try:
|
|
55
|
+
self._producer.produce(topic, msg, **produce_kwargs)
|
|
56
|
+
break
|
|
57
|
+
except BufferError:
|
|
58
|
+
# It's hard to know what the size limit on the buffer is, so we will try to
|
|
59
|
+
# wait until the number of items in it decreases (or it is empty, in case all
|
|
60
|
+
# messages were successfully sent in the time it took to handle the error).
|
|
61
|
+
buffer_len = len(self._producer)
|
|
62
|
+
self.logger.debug(f"Blocking due to BufferError, buffer size: {buffer_len}")
|
|
63
|
+
while buffer_len > 0 and len(self._producer) == buffer_len:
|
|
64
|
+
self._producer.poll(0.01)
|
|
65
|
+
|
|
66
|
+
def queued_message_count(self):
|
|
67
|
+
"""Get the number of messages waiting to be sent to the broker.
|
|
68
|
+
"""
|
|
69
|
+
return len(self._producer)
|
|
56
70
|
|
|
57
71
|
def flush(self, timeout: timedelta = timedelta(seconds=10)) -> int:
|
|
58
72
|
"""Attempt to flush enqueued messages. Return the number of messages still
|
|
@@ -66,9 +80,9 @@ class Producer:
|
|
|
66
80
|
self.logger.debug("flushed all messages")
|
|
67
81
|
return n
|
|
68
82
|
|
|
69
|
-
def close(self) -> int:
|
|
83
|
+
def close(self, timeout: timedelta = timedelta(seconds=10)) -> int:
|
|
70
84
|
self.logger.debug("shutting down producer")
|
|
71
|
-
return self.flush()
|
|
85
|
+
return self.flush(timeout)
|
|
72
86
|
|
|
73
87
|
def __enter__(self) -> 'Producer':
|
|
74
88
|
return self
|
|
@@ -142,7 +156,6 @@ class ProducerConfig:
|
|
|
142
156
|
config["error_cb"] = self.error_callback
|
|
143
157
|
if self.auth is not None:
|
|
144
158
|
config.update(self.auth())
|
|
145
|
-
set_oauth_cb(config)
|
|
146
159
|
return config
|
|
147
160
|
|
|
148
161
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: adc-streaming
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Astronomy Data Commons streaming client libraries
|
|
5
5
|
Home-page: https://github.com/astronomy-commons/adc-streaming
|
|
6
6
|
Author: Astronomy Data Commons Team
|
|
@@ -12,8 +12,35 @@ Classifier: Development Status :: 3 - Alpha
|
|
|
12
12
|
Classifier: Operating System :: POSIX :: Linux
|
|
13
13
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
14
14
|
Description-Content-Type: text/markdown
|
|
15
|
-
Provides-Extra: dev
|
|
16
15
|
License-File: LICENSE
|
|
16
|
+
Requires-Dist: confluent-kafka>=2.11.0
|
|
17
|
+
Requires-Dist: dataclasses; python_version < "3.7"
|
|
18
|
+
Requires-Dist: importlib-metadata; python_version < "3.8"
|
|
19
|
+
Requires-Dist: tqdm
|
|
20
|
+
Requires-Dist: certifi>=2020.04.05.1
|
|
21
|
+
Requires-Dist: typing-extensions; python_version < "3.8"
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: autopep8; extra == "dev"
|
|
24
|
+
Requires-Dist: docker; extra == "dev"
|
|
25
|
+
Requires-Dist: flake8; extra == "dev"
|
|
26
|
+
Requires-Dist: isort; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-timeout; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-integration; extra == "dev"
|
|
30
|
+
Requires-Dist: sphinx; extra == "dev"
|
|
31
|
+
Requires-Dist: sphinx_rtd_theme; extra == "dev"
|
|
32
|
+
Requires-Dist: twine; extra == "dev"
|
|
33
|
+
Dynamic: author
|
|
34
|
+
Dynamic: author-email
|
|
35
|
+
Dynamic: classifier
|
|
36
|
+
Dynamic: description
|
|
37
|
+
Dynamic: description-content-type
|
|
38
|
+
Dynamic: home-page
|
|
39
|
+
Dynamic: license
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
Dynamic: provides-extra
|
|
42
|
+
Dynamic: requires-dist
|
|
43
|
+
Dynamic: summary
|
|
17
44
|
|
|
18
45
|
# Astronomy Data Commons Streaming Client Libraries
|
|
19
46
|
|
|
@@ -12,13 +12,14 @@ adc/consumer.py
|
|
|
12
12
|
adc/errors.py
|
|
13
13
|
adc/io.py
|
|
14
14
|
adc/kafka.py
|
|
15
|
-
adc/oidc.py
|
|
16
15
|
adc/producer.py
|
|
17
16
|
adc_streaming.egg-info/PKG-INFO
|
|
18
17
|
adc_streaming.egg-info/SOURCES.txt
|
|
19
18
|
adc_streaming.egg-info/dependency_links.txt
|
|
20
19
|
adc_streaming.egg-info/not-zip-safe
|
|
21
20
|
adc_streaming.egg-info/requires.txt
|
|
21
|
+
adc_streaming.egg-info/scm_file_list.json
|
|
22
|
+
adc_streaming.egg-info/scm_version.json
|
|
22
23
|
adc_streaming.egg-info/top_level.txt
|
|
23
24
|
doc/Makefile
|
|
24
25
|
doc/conf.py
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".github/workflows/build.yml",
|
|
4
|
+
".gitignore",
|
|
5
|
+
"LICENSE",
|
|
6
|
+
"Makefile",
|
|
7
|
+
"README.md",
|
|
8
|
+
"adc/__init__.py",
|
|
9
|
+
"adc/auth.py",
|
|
10
|
+
"adc/consumer.py",
|
|
11
|
+
"adc/errors.py",
|
|
12
|
+
"adc/io.py",
|
|
13
|
+
"adc/kafka.py",
|
|
14
|
+
"adc/producer.py",
|
|
15
|
+
"doc/Makefile",
|
|
16
|
+
"doc/_static/css/my_theme.css",
|
|
17
|
+
"doc/_templates/layout.html",
|
|
18
|
+
"doc/api/api.rst",
|
|
19
|
+
"doc/api/streaming.rst",
|
|
20
|
+
"doc/conf.py",
|
|
21
|
+
"doc/index.rst",
|
|
22
|
+
"doc/user/installation.rst",
|
|
23
|
+
"doc/user/quickstart.rst",
|
|
24
|
+
"pyproject.toml",
|
|
25
|
+
"recipe/meta.yaml",
|
|
26
|
+
"setup.cfg",
|
|
27
|
+
"setup.py",
|
|
28
|
+
"tests/test_auth.py",
|
|
29
|
+
"tests/test_kafka.py",
|
|
30
|
+
"tests/test_kafka_integration.py"
|
|
31
|
+
]
|
|
32
|
+
}
|
|
@@ -3,11 +3,9 @@ from setuptools import setup
|
|
|
3
3
|
|
|
4
4
|
# requirements
|
|
5
5
|
install_requires = [
|
|
6
|
-
"
|
|
7
|
-
"confluent-kafka >= 1.6.1, != 2.1.0, != 2.1.1",
|
|
6
|
+
"confluent-kafka >= 2.11.0",
|
|
8
7
|
"dataclasses ; python_version < '3.7'",
|
|
9
8
|
"importlib-metadata ; python_version < '3.8'",
|
|
10
|
-
"requests", # FIXME: drop after next release of confluent-kafka with OIDC support
|
|
11
9
|
"tqdm",
|
|
12
10
|
"certifi>=2020.04.05.1",
|
|
13
11
|
"typing-extensions ; python_version < '3.8'",
|
|
@@ -28,7 +28,16 @@ from adc.auth import SASLAuth
|
|
|
28
28
|
'sasl.password': 'test-pass',
|
|
29
29
|
'security.protocol': 'SASL_SSL'
|
|
30
30
|
}
|
|
31
|
-
]
|
|
31
|
+
],
|
|
32
|
+
[
|
|
33
|
+
SASLAuth(
|
|
34
|
+
'test', '', token_command="generate_token"
|
|
35
|
+
),
|
|
36
|
+
{
|
|
37
|
+
'sasl.mechanism': 'OAUTHBEARER',
|
|
38
|
+
'sasl.oauthbearer.method': 'default',
|
|
39
|
+
}
|
|
40
|
+
],
|
|
32
41
|
])
|
|
33
42
|
def test_auth(auth, expected_config):
|
|
34
43
|
# Check that the key/value pairs in expected_config are a subset
|
|
@@ -5,7 +5,11 @@ import unittest
|
|
|
5
5
|
from datetime import datetime, timedelta
|
|
6
6
|
from typing import List, Optional
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
try:
|
|
9
|
+
import docker
|
|
10
|
+
have_docker = True
|
|
11
|
+
except ImportError:
|
|
12
|
+
have_docker = False
|
|
9
13
|
import pytest
|
|
10
14
|
|
|
11
15
|
import adc.consumer
|
|
@@ -17,6 +21,7 @@ logging.getLogger("adc-streaming").setLevel(logging.DEBUG)
|
|
|
17
21
|
logger = logging.getLogger("adc-streaming.tests")
|
|
18
22
|
|
|
19
23
|
|
|
24
|
+
@pytest.mark.skipif(not have_docker, reason="requires docker")
|
|
20
25
|
@pytest.mark.integration_test
|
|
21
26
|
class KafkaIntegrationTestCase(unittest.TestCase):
|
|
22
27
|
"""This test runs a Kafka broker in a Docker container, and makes sure that
|
|
@@ -84,6 +89,44 @@ class KafkaIntegrationTestCase(unittest.TestCase):
|
|
|
84
89
|
self.assertEqual(msg.value(), b"can you hear me?")
|
|
85
90
|
self.assertEqual(msg.key(), b"test_msg")
|
|
86
91
|
|
|
92
|
+
def test_message_with_callback(self):
|
|
93
|
+
"""Try writing a message into the Kafka broker, with a delivery callback.
|
|
94
|
+
"""
|
|
95
|
+
topic = "test_message_with_callback"
|
|
96
|
+
callback_called = False
|
|
97
|
+
callback_error = False
|
|
98
|
+
|
|
99
|
+
def callback(err, msg):
|
|
100
|
+
nonlocal callback_called
|
|
101
|
+
nonlocal callback_error
|
|
102
|
+
callback_called = True
|
|
103
|
+
callback_error = err is not None
|
|
104
|
+
|
|
105
|
+
ap = adc.producer
|
|
106
|
+
with ap.Producer(ap.ProducerConfig(broker_urls=[self.kafka.address],
|
|
107
|
+
topic=topic, auth=self.kafka.auth)) as producer:
|
|
108
|
+
producer.write("message data", delivery_callback=callback)
|
|
109
|
+
producer.flush()
|
|
110
|
+
consumer = adc.consumer.Consumer(adc.consumer.ConsumerConfig(
|
|
111
|
+
broker_urls=[self.kafka.address],
|
|
112
|
+
group_id="test_consumer",
|
|
113
|
+
auth=self.kafka.auth,
|
|
114
|
+
))
|
|
115
|
+
consumer.subscribe(topic)
|
|
116
|
+
stream = consumer.stream()
|
|
117
|
+
|
|
118
|
+
msg = next(stream)
|
|
119
|
+
if msg.error() is not None:
|
|
120
|
+
raise Exception(msg.error())
|
|
121
|
+
# give the producer's background thread time to notice the ack from the broker and
|
|
122
|
+
# fire the callback
|
|
123
|
+
producer._producer.poll(0.1)
|
|
124
|
+
|
|
125
|
+
self.assertEqual(msg.topic(), topic)
|
|
126
|
+
self.assertEqual(msg.value(), b"message data")
|
|
127
|
+
self.assertEqual(callback_called, True)
|
|
128
|
+
self.assertEqual(callback_error, False)
|
|
129
|
+
|
|
87
130
|
def test_reset_to_end(self):
|
|
88
131
|
# Write a few messages.
|
|
89
132
|
topic = "test_reset_to_end"
|
|
@@ -486,7 +529,7 @@ class KafkaDockerConnection:
|
|
|
486
529
|
"""Block until the Docker daemon tells us the IP and Port of the Kafa broker.
|
|
487
530
|
|
|
488
531
|
Returns the ip and port as a string in the form "ip:port."
|
|
489
|
-
"""
|
|
532
|
+
"""
|
|
490
533
|
i = 0
|
|
491
534
|
while (not self.query_kafka_broker_address()) and i < maxiter:
|
|
492
535
|
logger.info("polling to wait for container to acquire port...")
|
|
@@ -508,7 +551,7 @@ class KafkaDockerConnection:
|
|
|
508
551
|
port = addrs[0]['HostPort']
|
|
509
552
|
return f"{ip}:{port}"
|
|
510
553
|
|
|
511
|
-
def poll_for_kafka_active(self, maxiter=
|
|
554
|
+
def poll_for_kafka_active(self, maxiter=40, sleep=timedelta(milliseconds=500)):
|
|
512
555
|
"""Block until Kafka's network listener is accepting connections."""
|
|
513
556
|
i = 0
|
|
514
557
|
while (not self.query_kafka_active()) and i < maxiter:
|
adc-streaming-2.5.0/adc/auth.py
DELETED
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python
|
|
2
|
-
|
|
3
|
-
from enum import Enum
|
|
4
|
-
|
|
5
|
-
import certifi
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class SASLMethod(Enum):
|
|
9
|
-
"""SASL method to use for authentication.
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
PLAIN = 1
|
|
13
|
-
SCRAM_SHA_256 = 2
|
|
14
|
-
SCRAM_SHA_512 = 3
|
|
15
|
-
OAUTHBEARER = 4
|
|
16
|
-
|
|
17
|
-
def __str__(self):
|
|
18
|
-
return self.name.replace("_", "-")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class SASLAuth(object):
|
|
22
|
-
"""Attach SASL-based authentication to a client.
|
|
23
|
-
|
|
24
|
-
Returns client-based auth options when called.
|
|
25
|
-
|
|
26
|
-
Parameters
|
|
27
|
-
----------
|
|
28
|
-
user : `str`
|
|
29
|
-
Username to authenticate with.
|
|
30
|
-
password : `str`
|
|
31
|
-
Password to authenticate with.
|
|
32
|
-
ssl : `bool`, optional
|
|
33
|
-
Whether to enable SSL (enabled by default).
|
|
34
|
-
method : `SASLMethod`, optional
|
|
35
|
-
The SASL method to authenticate. The default is SASLMethod.OAUTHBEARER
|
|
36
|
-
if token_endpoint is provided, or SASLMethod.PLAIN otherwise.
|
|
37
|
-
See valid SASL methods in SASLMethod.
|
|
38
|
-
ssl_ca_location : `str`, optional
|
|
39
|
-
If using SSL via a self-signed cert, a path/location
|
|
40
|
-
to the certificate.
|
|
41
|
-
ssl_endpoint_identification_algorithm : `str`, optional
|
|
42
|
-
If using SSL, the algorithm used to verify that certificate is valid for the endpoint.
|
|
43
|
-
token_endpoint : `str`, optional
|
|
44
|
-
The OpenID Connect token endpoint URL.
|
|
45
|
-
Required for OAUTHBEARER / OpenID Connect, otherwise ignored.
|
|
46
|
-
|
|
47
|
-
"""
|
|
48
|
-
|
|
49
|
-
def __init__(self, user, password, ssl=True, method=None, token_endpoint=None, **kwargs):
|
|
50
|
-
if method is None:
|
|
51
|
-
if token_endpoint is None:
|
|
52
|
-
method = SASLMethod.PLAIN
|
|
53
|
-
else:
|
|
54
|
-
method = SASLMethod.OAUTHBEARER
|
|
55
|
-
self._method = method
|
|
56
|
-
|
|
57
|
-
# set up SSL options
|
|
58
|
-
if ssl:
|
|
59
|
-
if "ssl_ca_location" in kwargs:
|
|
60
|
-
ssl_cert = kwargs["ssl_ca_location"]
|
|
61
|
-
else:
|
|
62
|
-
ssl_cert = certifi.where()
|
|
63
|
-
|
|
64
|
-
self._config = {
|
|
65
|
-
"security.protocol": "SASL_SSL",
|
|
66
|
-
"ssl.ca.location": ssl_cert,
|
|
67
|
-
}
|
|
68
|
-
if "ssl_endpoint_identification_algorithm" in kwargs:
|
|
69
|
-
self._config["ssl.endpoint.identification.algorithm"] = \
|
|
70
|
-
kwargs["ssl_endpoint_identification_algorithm"]
|
|
71
|
-
else:
|
|
72
|
-
self._config = {"security.protocol": "SASL_PLAINTEXT"}
|
|
73
|
-
|
|
74
|
-
# set up SASL options
|
|
75
|
-
self._config["sasl.mechanism"] = str(self._method)
|
|
76
|
-
if token_endpoint:
|
|
77
|
-
self._config["sasl.oauthbearer.client.id"] = user
|
|
78
|
-
self._config["sasl.oauthbearer.client.secret"] = password
|
|
79
|
-
self._config["sasl.oauthbearer.method"] = "oidc"
|
|
80
|
-
self._config["sasl.oauthbearer.token.endpoint.url"] = token_endpoint
|
|
81
|
-
else:
|
|
82
|
-
self._config["sasl.username"] = user
|
|
83
|
-
self._config["sasl.password"] = password
|
|
84
|
-
|
|
85
|
-
def __call__(self):
|
|
86
|
-
return self._config
|
adc-streaming-2.5.0/adc/oidc.py
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
def set_oauth_cb(config):
|
|
2
|
-
"""Implement client support for KIP-768 OpenID Connect.
|
|
3
|
-
|
|
4
|
-
Apache Kafka 3.1.0 supports authentication using OpenID Client Credentials.
|
|
5
|
-
Native support for Python is still incomplete due to this issue:
|
|
6
|
-
https://github.com/confluentinc/librdkafka/issues/3751
|
|
7
|
-
|
|
8
|
-
Meanwhile, this is a pure Python implementation of the refresh token
|
|
9
|
-
callback.
|
|
10
|
-
"""
|
|
11
|
-
if config.pop('sasl.oauthbearer.method', None) != 'oidc':
|
|
12
|
-
return
|
|
13
|
-
|
|
14
|
-
client_id = config.pop('sasl.oauthbearer.client.id')
|
|
15
|
-
client_secret = config.pop('sasl.oauthbearer.client.secret')
|
|
16
|
-
scope = config.pop('sasl.oauthbearer.scope', None)
|
|
17
|
-
token_endpoint = config.pop('sasl.oauthbearer.token.endpoint.url')
|
|
18
|
-
|
|
19
|
-
from authlib.integrations.requests_client import OAuth2Session
|
|
20
|
-
session = OAuth2Session(client_id, client_secret, scope=scope)
|
|
21
|
-
|
|
22
|
-
def oauth_cb(*_, **__):
|
|
23
|
-
token = session.fetch_token(
|
|
24
|
-
token_endpoint, grant_type='client_credentials')
|
|
25
|
-
return token['access_token'], token['expires_at']
|
|
26
|
-
|
|
27
|
-
config['oauth_cb'] = oauth_cb
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|