kafka-python 2.3.0__py2.py3-none-any.whl → 2.3.2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka/admin/client.py +4 -2
- kafka/client_async.py +4 -0
- kafka/codec.py +2 -4
- kafka/conn.py +16 -8
- kafka/consumer/fetcher.py +5 -4
- kafka/consumer/group.py +4 -2
- kafka/coordinator/consumer.py +2 -1
- kafka/errors.py +4 -0
- kafka/future.py +8 -0
- kafka/producer/kafka.py +4 -2
- kafka/protocol/parser.py +9 -1
- kafka/protocol/types.py +21 -20
- kafka/sasl/scram.py +6 -1
- kafka/version.py +1 -1
- {kafka_python-2.3.0.dist-info → kafka_python-2.3.2.dist-info}/METADATA +23 -26
- {kafka_python-2.3.0.dist-info → kafka_python-2.3.2.dist-info}/RECORD +18 -18
- {kafka_python-2.3.0.dist-info → kafka_python-2.3.2.dist-info}/WHEEL +1 -1
- {kafka_python-2.3.0.dist-info → kafka_python-2.3.2.dist-info}/top_level.txt +0 -0
kafka/admin/client.py
CHANGED
|
@@ -88,6 +88,9 @@ class KafkaAdminClient(object):
|
|
|
88
88
|
max_in_flight_requests_per_connection (int): Requests are pipelined
|
|
89
89
|
to kafka brokers up to this number of maximum requests per
|
|
90
90
|
broker connection. Default: 5.
|
|
91
|
+
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
92
|
+
Used to avoid OOM when decoding malformed network message header.
|
|
93
|
+
Default: 1000000.
|
|
91
94
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
92
95
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
93
96
|
system defaults). Java client defaults to 32768.
|
|
@@ -164,11 +167,10 @@ class KafkaAdminClient(object):
|
|
|
164
167
|
'reconnect_backoff_ms': 50,
|
|
165
168
|
'reconnect_backoff_max_ms': 30000,
|
|
166
169
|
'max_in_flight_requests_per_connection': 5,
|
|
170
|
+
'receive_message_max_bytes': 1000000,
|
|
167
171
|
'receive_buffer_bytes': None,
|
|
168
172
|
'send_buffer_bytes': None,
|
|
169
173
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
|
170
|
-
'sock_chunk_bytes': 4096, # undocumented experimental option
|
|
171
|
-
'sock_chunk_buffer_count': 1000, # undocumented experimental option
|
|
172
174
|
'retry_backoff_ms': 100,
|
|
173
175
|
'metadata_max_age_ms': 300000,
|
|
174
176
|
'security_protocol': 'PLAINTEXT',
|
kafka/client_async.py
CHANGED
|
@@ -89,6 +89,9 @@ class KafkaClient(object):
|
|
|
89
89
|
max_in_flight_requests_per_connection (int): Requests are pipelined
|
|
90
90
|
to kafka brokers up to this number of maximum requests per
|
|
91
91
|
broker connection. Default: 5.
|
|
92
|
+
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
93
|
+
Used to avoid OOM when decoding malformed network message header.
|
|
94
|
+
Default: 1000000.
|
|
92
95
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
93
96
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
94
97
|
system defaults). Java client defaults to 32768.
|
|
@@ -186,6 +189,7 @@ class KafkaClient(object):
|
|
|
186
189
|
'reconnect_backoff_ms': 50,
|
|
187
190
|
'reconnect_backoff_max_ms': 30000,
|
|
188
191
|
'max_in_flight_requests_per_connection': 5,
|
|
192
|
+
'receive_message_max_bytes': 1000000,
|
|
189
193
|
'receive_buffer_bytes': None,
|
|
190
194
|
'send_buffer_bytes': None,
|
|
191
195
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
kafka/codec.py
CHANGED
|
@@ -327,7 +327,5 @@ def zstd_encode(payload):
|
|
|
327
327
|
def zstd_decode(payload):
|
|
328
328
|
if not zstd:
|
|
329
329
|
raise NotImplementedError("Zstd codec is not available")
|
|
330
|
-
|
|
331
|
-
return
|
|
332
|
-
except zstd.ZstdError:
|
|
333
|
-
return zstd.ZstdDecompressor().decompress(payload, max_output_size=ZSTD_MAX_OUTPUT_SIZE)
|
|
330
|
+
with zstd.ZstdDecompressor().stream_reader(io.BytesIO(payload), read_across_frames=True) as reader:
|
|
331
|
+
return reader.read()
|
kafka/conn.py
CHANGED
|
@@ -122,6 +122,9 @@ class BrokerConnection(object):
|
|
|
122
122
|
max_in_flight_requests_per_connection (int): Requests are pipelined
|
|
123
123
|
to kafka brokers up to this number of maximum requests per
|
|
124
124
|
broker connection. Default: 5.
|
|
125
|
+
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
126
|
+
Used to avoid OOM when decoding malformed network message header.
|
|
127
|
+
Default: 1000000.
|
|
125
128
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
126
129
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
127
130
|
system defaults). Java client defaults to 32768.
|
|
@@ -202,6 +205,7 @@ class BrokerConnection(object):
|
|
|
202
205
|
'reconnect_backoff_ms': 50,
|
|
203
206
|
'reconnect_backoff_max_ms': 30000,
|
|
204
207
|
'max_in_flight_requests_per_connection': 5,
|
|
208
|
+
'receive_message_max_bytes': 1000000,
|
|
205
209
|
'receive_buffer_bytes': None,
|
|
206
210
|
'send_buffer_bytes': None,
|
|
207
211
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
|
@@ -292,7 +296,8 @@ class BrokerConnection(object):
|
|
|
292
296
|
|
|
293
297
|
self._protocol = KafkaProtocol(
|
|
294
298
|
client_id=self.config['client_id'],
|
|
295
|
-
api_version=self.config['api_version']
|
|
299
|
+
api_version=self.config['api_version'],
|
|
300
|
+
max_frame_size=self.config['receive_message_max_bytes'])
|
|
296
301
|
self.state = ConnectionStates.DISCONNECTED
|
|
297
302
|
self._reset_reconnect_backoff()
|
|
298
303
|
self._sock = None
|
|
@@ -695,9 +700,12 @@ class BrokerConnection(object):
|
|
|
695
700
|
'Kafka broker does not support %s sasl mechanism. Enabled mechanisms are: %s'
|
|
696
701
|
% (self.config['sasl_mechanism'], response.enabled_mechanisms)))
|
|
697
702
|
else:
|
|
698
|
-
|
|
703
|
+
try:
|
|
704
|
+
ret = self._sasl_authenticate()
|
|
705
|
+
future.success(ret)
|
|
706
|
+
except Exception as exc:
|
|
707
|
+
future.failure(exc)
|
|
699
708
|
|
|
700
|
-
assert future.is_done, 'SASL future not complete after mechanism processing!'
|
|
701
709
|
if future.failed():
|
|
702
710
|
self.close(error=future.exception)
|
|
703
711
|
else:
|
|
@@ -809,24 +817,24 @@ class BrokerConnection(object):
|
|
|
809
817
|
log.debug('%s: Received %d raw sasl auth bytes from server', self, nbytes)
|
|
810
818
|
return data[4:]
|
|
811
819
|
|
|
812
|
-
def _sasl_authenticate(self
|
|
820
|
+
def _sasl_authenticate(self):
|
|
813
821
|
while not self._sasl_mechanism.is_done():
|
|
814
822
|
send_token = self._sasl_mechanism.auth_bytes()
|
|
815
823
|
self._send_sasl_authenticate(send_token)
|
|
816
824
|
if not self._can_send_recv():
|
|
817
|
-
|
|
825
|
+
raise Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self)
|
|
818
826
|
|
|
819
827
|
recv_token = self._recv_sasl_authenticate()
|
|
820
828
|
if recv_token is None:
|
|
821
|
-
|
|
829
|
+
raise Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self)
|
|
822
830
|
else:
|
|
823
831
|
self._sasl_mechanism.receive(recv_token)
|
|
824
832
|
|
|
825
833
|
if self._sasl_mechanism.is_authenticated():
|
|
826
834
|
log.info('%s: %s', self, self._sasl_mechanism.auth_details())
|
|
827
|
-
return
|
|
835
|
+
return True
|
|
828
836
|
else:
|
|
829
|
-
|
|
837
|
+
raise Errors.SaslAuthenticationFailedError('Failed to authenticate via SASL %s' % self.config['sasl_mechanism'])
|
|
830
838
|
|
|
831
839
|
def blacked_out(self):
|
|
832
840
|
"""
|
kafka/consumer/fetcher.py
CHANGED
|
@@ -250,16 +250,17 @@ class Fetcher(six.Iterator):
|
|
|
250
250
|
break
|
|
251
251
|
|
|
252
252
|
if future.succeeded():
|
|
253
|
-
|
|
254
|
-
|
|
253
|
+
offsets, retry = future.value
|
|
254
|
+
fetched_offsets.update(offsets)
|
|
255
|
+
if not retry:
|
|
255
256
|
return fetched_offsets
|
|
256
257
|
|
|
257
|
-
timestamps = {tp: timestamps[tp] for tp in
|
|
258
|
+
timestamps = {tp: timestamps[tp] for tp in retry}
|
|
258
259
|
|
|
259
260
|
elif not future.retriable():
|
|
260
261
|
raise future.exception # pylint: disable-msg=raising-bad-type
|
|
261
262
|
|
|
262
|
-
|
|
263
|
+
elif future.exception.invalid_metadata or self._client.cluster.need_update:
|
|
263
264
|
refresh_future = self._client.cluster.request_update()
|
|
264
265
|
self._client.poll(future=refresh_future, timeout_ms=timer.timeout_ms)
|
|
265
266
|
|
kafka/consumer/group.py
CHANGED
|
@@ -172,6 +172,9 @@ class KafkaConsumer(six.Iterator):
|
|
|
172
172
|
should be set no higher than 1/3 of that value. It can be
|
|
173
173
|
adjusted even lower to control the expected time for normal
|
|
174
174
|
rebalances. Default: 3000
|
|
175
|
+
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
176
|
+
Used to avoid OOM when decoding malformed network message header.
|
|
177
|
+
Default: 1000000.
|
|
175
178
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
176
179
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
177
180
|
system defaults). The java client defaults to 32768.
|
|
@@ -313,9 +316,8 @@ class KafkaConsumer(six.Iterator):
|
|
|
313
316
|
'heartbeat_interval_ms': 3000,
|
|
314
317
|
'receive_buffer_bytes': None,
|
|
315
318
|
'send_buffer_bytes': None,
|
|
319
|
+
'receive_message_max_bytes': 1000000,
|
|
316
320
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
|
317
|
-
'sock_chunk_bytes': 4096, # undocumented experimental option
|
|
318
|
-
'sock_chunk_buffer_count': 1000, # undocumented experimental option
|
|
319
321
|
'consumer_timeout_ms': float('inf'),
|
|
320
322
|
'security_protocol': 'PLAINTEXT',
|
|
321
323
|
'ssl_context': None,
|
kafka/coordinator/consumer.py
CHANGED
|
@@ -457,7 +457,8 @@ class ConsumerCoordinator(BaseCoordinator):
|
|
|
457
457
|
self._client.poll(future=future, timeout_ms=timer.timeout_ms)
|
|
458
458
|
|
|
459
459
|
if future.is_done:
|
|
460
|
-
|
|
460
|
+
if future_key in self._offset_fetch_futures:
|
|
461
|
+
del self._offset_fetch_futures[future_key]
|
|
461
462
|
|
|
462
463
|
if future.succeeded():
|
|
463
464
|
return future.value
|
kafka/errors.py
CHANGED
kafka/future.py
CHANGED
|
@@ -37,6 +37,10 @@ class Future(object):
|
|
|
37
37
|
self.is_done = True
|
|
38
38
|
if self._callbacks:
|
|
39
39
|
self._call_backs('callback', self._callbacks, self.value)
|
|
40
|
+
# Clearing the lists releases any reference cycle held via stored
|
|
41
|
+
# bound methods (e.g. FutureProduceResult<->FutureRecordMetadata).
|
|
42
|
+
self._callbacks = None
|
|
43
|
+
self._errbacks = None
|
|
40
44
|
return self
|
|
41
45
|
|
|
42
46
|
def failure(self, e):
|
|
@@ -48,6 +52,10 @@ class Future(object):
|
|
|
48
52
|
self.exception = exception
|
|
49
53
|
self.is_done = True
|
|
50
54
|
self._call_backs('errback', self._errbacks, self.exception)
|
|
55
|
+
# Clearing the lists releases any reference cycle held via stored
|
|
56
|
+
# bound methods (e.g. FutureProduceResult<->FutureRecordMetadata).
|
|
57
|
+
self._callbacks = None
|
|
58
|
+
self._errbacks = None
|
|
51
59
|
return self
|
|
52
60
|
|
|
53
61
|
def add_callback(self, f, *args, **kwargs):
|
kafka/producer/kafka.py
CHANGED
|
@@ -261,6 +261,9 @@ class KafkaProducer(object):
|
|
|
261
261
|
errors. Default: 100.
|
|
262
262
|
request_timeout_ms (int): Client request timeout in milliseconds.
|
|
263
263
|
Default: 30000.
|
|
264
|
+
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
265
|
+
Used to avoid OOM when decoding malformed network message header.
|
|
266
|
+
Default: 1000000.
|
|
264
267
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
265
268
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
266
269
|
system defaults). Java client defaults to 32768.
|
|
@@ -392,11 +395,10 @@ class KafkaProducer(object):
|
|
|
392
395
|
'metadata_max_age_ms': 300000,
|
|
393
396
|
'retry_backoff_ms': 100,
|
|
394
397
|
'request_timeout_ms': 30000,
|
|
398
|
+
'receive_message_max_bytes': 1000000,
|
|
395
399
|
'receive_buffer_bytes': None,
|
|
396
400
|
'send_buffer_bytes': None,
|
|
397
401
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
|
398
|
-
'sock_chunk_bytes': 4096, # undocumented experimental option
|
|
399
|
-
'sock_chunk_buffer_count': 1000, # undocumented experimental option
|
|
400
402
|
'reconnect_backoff_ms': 50,
|
|
401
403
|
'reconnect_backoff_max_ms': 30000,
|
|
402
404
|
'max_in_flight_requests_per_connection': 5,
|
kafka/protocol/parser.py
CHANGED
|
@@ -23,12 +23,15 @@ class KafkaProtocol(object):
|
|
|
23
23
|
api_version (tuple): Optional tuple to specify api_version to use.
|
|
24
24
|
Currently only used to check for 0.8.2 protocol quirks, but
|
|
25
25
|
may be used for more in the future.
|
|
26
|
+
max_frame_size (int): Maximum allowed message frame size.
|
|
27
|
+
Default: 100000000 (100MB).
|
|
26
28
|
"""
|
|
27
|
-
def __init__(self, client_id=None, api_version=None):
|
|
29
|
+
def __init__(self, client_id=None, api_version=None, max_frame_size=100000000):
|
|
28
30
|
if client_id is None:
|
|
29
31
|
client_id = self._gen_client_id()
|
|
30
32
|
self._client_id = client_id
|
|
31
33
|
self._api_version = api_version
|
|
34
|
+
self._max_frame_size = max_frame_size
|
|
32
35
|
self._correlation_id = 0
|
|
33
36
|
self._header = KafkaBytes(4)
|
|
34
37
|
self._rbuffer = None
|
|
@@ -105,6 +108,7 @@ class KafkaProtocol(object):
|
|
|
105
108
|
if self._header.tell() == 4:
|
|
106
109
|
self._header.seek(0)
|
|
107
110
|
nbytes = Int32.decode(self._header)
|
|
111
|
+
self._validate_frame_size(nbytes)
|
|
108
112
|
# reset buffer and switch state to receiving payload bytes
|
|
109
113
|
self._rbuffer = KafkaBytes(nbytes)
|
|
110
114
|
self._receiving = True
|
|
@@ -132,6 +136,10 @@ class KafkaProtocol(object):
|
|
|
132
136
|
self._reset_buffer()
|
|
133
137
|
return responses
|
|
134
138
|
|
|
139
|
+
def _validate_frame_size(self, nbytes):
|
|
140
|
+
if nbytes < 0 or nbytes > self._max_frame_size:
|
|
141
|
+
raise Errors.InvalidReceiveError('Invalid frame length: %d' % nbytes)
|
|
142
|
+
|
|
135
143
|
def _process_response(self, read_buffer):
|
|
136
144
|
if not self.in_flight_requests:
|
|
137
145
|
raise Errors.CorrelationIdError('No in-flight-request found for server response')
|
kafka/protocol/types.py
CHANGED
|
@@ -213,6 +213,17 @@ class Array(AbstractType):
|
|
|
213
213
|
|
|
214
214
|
|
|
215
215
|
class UnsignedVarInt32(AbstractType):
|
|
216
|
+
@classmethod
|
|
217
|
+
def decode(cls, data):
|
|
218
|
+
value = VarInt32.decode(data)
|
|
219
|
+
return (value << 1) ^ (value >> 31)
|
|
220
|
+
|
|
221
|
+
@classmethod
|
|
222
|
+
def encode(cls, value):
|
|
223
|
+
return VarInt32.encode((value >> 1) ^ -(value & 1))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class VarInt32(AbstractType):
|
|
216
227
|
@classmethod
|
|
217
228
|
def decode(cls, data):
|
|
218
229
|
value, i = 0, 0
|
|
@@ -225,10 +236,12 @@ class UnsignedVarInt32(AbstractType):
|
|
|
225
236
|
if i > 28:
|
|
226
237
|
raise ValueError('Invalid value {}'.format(value))
|
|
227
238
|
value |= b << i
|
|
228
|
-
return value
|
|
239
|
+
return (value >> 1) ^ -(value & 1)
|
|
229
240
|
|
|
230
241
|
@classmethod
|
|
231
242
|
def encode(cls, value):
|
|
243
|
+
# bring it in line with the java binary repr
|
|
244
|
+
value = (value << 1) ^ (value >> 31)
|
|
232
245
|
value &= 0xffffffff
|
|
233
246
|
ret = b''
|
|
234
247
|
while (value & 0xffffff80) != 0:
|
|
@@ -239,25 +252,12 @@ class UnsignedVarInt32(AbstractType):
|
|
|
239
252
|
return ret
|
|
240
253
|
|
|
241
254
|
|
|
242
|
-
class VarInt32(AbstractType):
|
|
243
|
-
@classmethod
|
|
244
|
-
def decode(cls, data):
|
|
245
|
-
value = UnsignedVarInt32.decode(data)
|
|
246
|
-
return (value >> 1) ^ -(value & 1)
|
|
247
|
-
|
|
248
|
-
@classmethod
|
|
249
|
-
def encode(cls, value):
|
|
250
|
-
# bring it in line with the java binary repr
|
|
251
|
-
value &= 0xffffffff
|
|
252
|
-
return UnsignedVarInt32.encode((value << 1) ^ (value >> 31))
|
|
253
|
-
|
|
254
|
-
|
|
255
255
|
class VarInt64(AbstractType):
|
|
256
256
|
@classmethod
|
|
257
257
|
def decode(cls, data):
|
|
258
258
|
value, i = 0, 0
|
|
259
259
|
while True:
|
|
260
|
-
b = data.read(1)
|
|
260
|
+
b, = struct.unpack('B', data.read(1))
|
|
261
261
|
if not (b & 0x80):
|
|
262
262
|
break
|
|
263
263
|
value |= (b & 0x7f) << i
|
|
@@ -270,14 +270,14 @@ class VarInt64(AbstractType):
|
|
|
270
270
|
@classmethod
|
|
271
271
|
def encode(cls, value):
|
|
272
272
|
# bring it in line with the java binary repr
|
|
273
|
+
value = (value << 1) ^ (value >> 63)
|
|
273
274
|
value &= 0xffffffffffffffff
|
|
274
|
-
v = (value << 1) ^ (value >> 63)
|
|
275
275
|
ret = b''
|
|
276
|
-
while (
|
|
276
|
+
while (value & 0xffffffffffffff80) != 0:
|
|
277
277
|
b = (value & 0x7f) | 0x80
|
|
278
278
|
ret += struct.pack('B', b)
|
|
279
|
-
|
|
280
|
-
ret += struct.pack('B',
|
|
279
|
+
value >>= 7
|
|
280
|
+
ret += struct.pack('B', value)
|
|
281
281
|
return ret
|
|
282
282
|
|
|
283
283
|
|
|
@@ -322,8 +322,9 @@ class TaggedFields(AbstractType):
|
|
|
322
322
|
for k, v in value.items():
|
|
323
323
|
# do we allow for other data types ?? It could get complicated really fast
|
|
324
324
|
assert isinstance(v, bytes), 'Value {} is not a byte array'.format(v)
|
|
325
|
-
assert isinstance(k, int) and k
|
|
325
|
+
assert isinstance(k, int) and k >= 0, 'Key {} is not a non-negative integer'.format(k)
|
|
326
326
|
ret += UnsignedVarInt32.encode(k)
|
|
327
|
+
ret += UnsignedVarInt32.encode(len(v))
|
|
327
328
|
ret += v
|
|
328
329
|
return ret
|
|
329
330
|
|
kafka/sasl/scram.py
CHANGED
|
@@ -106,7 +106,12 @@ class ScramClient:
|
|
|
106
106
|
self.auth_message += b',c=biws,r=' + self.nonce
|
|
107
107
|
|
|
108
108
|
salt = base64.b64decode(params['s'].encode('utf-8'))
|
|
109
|
-
|
|
109
|
+
try:
|
|
110
|
+
iterations = int(params['i'])
|
|
111
|
+
if iterations > 1000000:
|
|
112
|
+
raise ValueError('too many iterations')
|
|
113
|
+
except (TypeError, ValueError):
|
|
114
|
+
raise ValueError('Invalid value (not integer or too large) for Iteration count in server-first-message')
|
|
110
115
|
self.create_salted_password(salt, iterations)
|
|
111
116
|
|
|
112
117
|
self.client_key = self.hmac(self.salted_password, b'Client Key')
|
kafka/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = '2.3.
|
|
1
|
+
__version__ = '2.3.2'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
2
|
Name: kafka-python
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: Pure Python client for Apache Kafka
|
|
5
5
|
Author-email: Dana Powers <dana.powers@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/dpkp/kafka-python
|
|
@@ -26,21 +26,21 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
|
26
26
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
27
27
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
28
28
|
Description-Content-Type: text/x-rst
|
|
29
|
+
Provides-Extra: benchmarks
|
|
30
|
+
Requires-Dist: pyperf ; extra == 'benchmarks'
|
|
29
31
|
Provides-Extra: crc32c
|
|
30
|
-
Requires-Dist: crc32c; extra ==
|
|
32
|
+
Requires-Dist: crc32c ; extra == 'crc32c'
|
|
31
33
|
Provides-Extra: lz4
|
|
32
|
-
Requires-Dist: lz4; extra ==
|
|
34
|
+
Requires-Dist: lz4 ; extra == 'lz4'
|
|
33
35
|
Provides-Extra: snappy
|
|
34
|
-
Requires-Dist: python-snappy; extra ==
|
|
35
|
-
Provides-Extra: zstd
|
|
36
|
-
Requires-Dist: zstandard; extra == "zstd"
|
|
36
|
+
Requires-Dist: python-snappy ; extra == 'snappy'
|
|
37
37
|
Provides-Extra: testing
|
|
38
|
-
Requires-Dist: pytest; extra ==
|
|
39
|
-
Requires-Dist: mock;
|
|
40
|
-
Requires-Dist: pytest-
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Provides-Extra:
|
|
43
|
-
Requires-Dist:
|
|
38
|
+
Requires-Dist: pytest ; extra == 'testing'
|
|
39
|
+
Requires-Dist: pytest-mock ; extra == 'testing'
|
|
40
|
+
Requires-Dist: pytest-timeout ; extra == 'testing'
|
|
41
|
+
Requires-Dist: mock ; (python_version < "3.3") and extra == 'testing'
|
|
42
|
+
Provides-Extra: zstd
|
|
43
|
+
Requires-Dist: zstandard ; extra == 'zstd'
|
|
44
44
|
|
|
45
45
|
Kafka Python client
|
|
46
46
|
------------------------
|
|
@@ -66,22 +66,11 @@ Python client for the Apache Kafka distributed stream processing system.
|
|
|
66
66
|
kafka-python is designed to function much like the official java client, with a
|
|
67
67
|
sprinkling of pythonic interfaces (e.g., consumer iterators).
|
|
68
68
|
|
|
69
|
-
kafka-python is best used with newer brokers (0.9+), but is backwards-compatible with
|
|
70
|
-
older versions (to 0.8.0). Some features will only be enabled on newer brokers.
|
|
71
|
-
For example, fully coordinated consumer groups -- i.e., dynamic partition
|
|
72
|
-
assignment to multiple consumers in the same group -- requires use of 0.9+ kafka
|
|
73
|
-
brokers. Supporting this feature for earlier broker releases would require
|
|
74
|
-
writing and maintaining custom leadership election and membership / health
|
|
75
|
-
check code (perhaps using zookeeper or consul). For older brokers, you can
|
|
76
|
-
achieve something similar by manually assigning different partitions to each
|
|
77
|
-
consumer instance with config management tools like chef, ansible, etc. This
|
|
78
|
-
approach will work fine, though it does not support rebalancing on failures.
|
|
79
|
-
See https://kafka-python.readthedocs.io/en/master/compatibility.html
|
|
80
|
-
for more details.
|
|
81
|
-
|
|
82
69
|
Please note that the master branch may contain unreleased features. For release
|
|
83
70
|
documentation, please see readthedocs and/or python's inline help.
|
|
84
71
|
|
|
72
|
+
New in 2.3 release: python -m kafka.* interfaces for quick scripts and testing.
|
|
73
|
+
|
|
85
74
|
.. code-block:: bash
|
|
86
75
|
|
|
87
76
|
$ pip install kafka-python
|
|
@@ -232,6 +221,14 @@ for more details.
|
|
|
232
221
|
metrics = producer.metrics()
|
|
233
222
|
|
|
234
223
|
|
|
224
|
+
Module CLI Interface
|
|
225
|
+
********************
|
|
226
|
+
|
|
227
|
+
kafka-python also provides simple command-line interfaces for consumer, producer, and admin clients.
|
|
228
|
+
Access via ``python -m kafka.consumer``, ``python -m kafka.producer``, and ``python -m kafka.admin``.
|
|
229
|
+
See https://kafka-python.readthedocs.io/en/master/usage.html for more details.
|
|
230
|
+
|
|
231
|
+
|
|
235
232
|
Thread safety
|
|
236
233
|
*************
|
|
237
234
|
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
kafka/__init__.py,sha256=4dvHKZAxmD_4tfJ5wGcRV2X78vPcm8vsUoqceULevjA,1077
|
|
2
2
|
kafka/__main__.py,sha256=HNnJxekZZkNDXQwqnE5AfaM56JNCNYvq135wu4jxvhk,107
|
|
3
|
-
kafka/client_async.py,sha256=
|
|
3
|
+
kafka/client_async.py,sha256=BoZ9-wMPtYdk3Gvd2YfQNlPeLGn_BnHa_yR6Hcx7ZEI,57262
|
|
4
4
|
kafka/cluster.py,sha256=kTWqLHD-CZY_ua2LftVSvEjmNAL7KJ98Rd17zA177FU,16915
|
|
5
|
-
kafka/codec.py,sha256=
|
|
6
|
-
kafka/conn.py,sha256=
|
|
7
|
-
kafka/errors.py,sha256=
|
|
8
|
-
kafka/future.py,sha256=
|
|
5
|
+
kafka/codec.py,sha256=_jglwQ8yn-B85hzf0nvOvazRwGQSBx22Bb8R4sSCvcs,9977
|
|
6
|
+
kafka/conn.py,sha256=QjIvbI-oj2-rW4NYy5w3hxrR2aNPDqlQVdKT5vz_Bxw,70675
|
|
7
|
+
kafka/errors.py,sha256=PiyKR9l88dFBSCEdm91ryKXcOOsTsg88OGo-cndsV9k,33509
|
|
8
|
+
kafka/future.py,sha256=8GzUbOsLFHywit6Z-F2FjCXyoq0Jfa2K4JImFioQg_I,3267
|
|
9
9
|
kafka/socks5_wrapper.py,sha256=h0Gag3xAOp8io2MfzeYLOiNvLRmq3rkmpZ6Aj-9uKTw,10716
|
|
10
10
|
kafka/structs.py,sha256=SJGzmLdV21jZyQ7247k0WFy16UiusgTHK3I-e4qzI-E,3058
|
|
11
11
|
kafka/util.py,sha256=ncFqg0mXda_ipyLoAw_EF4dX8-lYanC2QtzC-l_F7-w,4585
|
|
12
|
-
kafka/version.py,sha256=
|
|
12
|
+
kafka/version.py,sha256=rEECwGgTKrdwQHjREbmsSsgmLKRQO-PCxUxDt3EApfA,22
|
|
13
13
|
kafka/admin/__init__.py,sha256=S_XxqyyV480_yXhttK79XZqNAmZyXRjspd3SoqYykE8,720
|
|
14
14
|
kafka/admin/__main__.py,sha256=mQrmllHNZx4SnFWd5WHJdQqfsqunFR1pnkIVppm3MqU,109
|
|
15
15
|
kafka/admin/acl_resource.py,sha256=UVfKDMsxRl-yGeCrbn0OX74aDbBQTibLgWMfCxkle5I,8480
|
|
16
|
-
kafka/admin/client.py,sha256=
|
|
16
|
+
kafka/admin/client.py,sha256=d6hIRnWaYAuEA0VuVvo_jdt_98RKdYyDV2Nf314T3Q4,73496
|
|
17
17
|
kafka/admin/config_resource.py,sha256=ZYKXRyHFcNVBlsyuL5435diW0Wgjd7R6Mbfi6TgDrfY,1252
|
|
18
18
|
kafka/admin/new_partitions.py,sha256=rYSb7S6VL706ZauSmiN5J9GDsep0HYRmkkAZUgT2JIg,757
|
|
19
19
|
kafka/admin/new_topic.py,sha256=NbtPrk6VaT-f-V8f3jT0s8m0uN-cvPfvOBYFEVgxSoA,1042
|
|
@@ -46,12 +46,12 @@ kafka/cli/consumer/__init__.py,sha256=Kw4v9vSLBXD2KxSSMrloAuauWi7Zke2C2M8PCvcGJ4
|
|
|
46
46
|
kafka/cli/producer/__init__.py,sha256=Wy_8p1TSzDkd9YnFvgG91AFXN6j8toVVfrNA0r_kplY,2633
|
|
47
47
|
kafka/consumer/__init__.py,sha256=NDdvtyuJgFyQZahqL9i5sYXGP6rOMIXWwHQEaZ1fCcs,122
|
|
48
48
|
kafka/consumer/__main__.py,sha256=JcySmP6wWi5NxK4icqMWPbFkf2722X4J-gQvbQZtDsk,112
|
|
49
|
-
kafka/consumer/fetcher.py,sha256=
|
|
50
|
-
kafka/consumer/group.py,sha256=
|
|
49
|
+
kafka/consumer/fetcher.py,sha256=TR6LJ2pa0LfJ5kTC-6DPHkMPO2CAo8qV-iO79VQ5F10,69375
|
|
50
|
+
kafka/consumer/group.py,sha256=EIHRR7V6iCdx9rKsYCyGn51Vz8E90inV1VbFHR0PvH0,60721
|
|
51
51
|
kafka/consumer/subscription_state.py,sha256=bK-YTVbOzhy8OB206QAfXVuo7zPA9YqYXnrRRST369c,24289
|
|
52
52
|
kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
53
|
kafka/coordinator/base.py,sha256=QP0YwJVj244BPwx9vP5XlI-E6geniztsJstiyglZI2k,58502
|
|
54
|
-
kafka/coordinator/consumer.py,sha256=
|
|
54
|
+
kafka/coordinator/consumer.py,sha256=ZHiiCKRsglPwhgQ5u7QPeFmwKDCRaJbA05ioZJITzWI,48574
|
|
55
55
|
kafka/coordinator/heartbeat.py,sha256=LeJJlwz1oUEOfEMIFT-R7ZOHBQ-b-luVKwmKyWxLfDo,3242
|
|
56
56
|
kafka/coordinator/protocol.py,sha256=Jq5OvRGGSrVCymufXsD-ZfUK-3ntsuFZE3Zu7lIaPkg,1041
|
|
57
57
|
kafka/coordinator/subscription.py,sha256=Rzdl2WnZdFB6v9M1cpZiWB_yzmVUJzB2X15KttMdBmw,922
|
|
@@ -92,7 +92,7 @@ kafka/partitioner/default.py,sha256=tW-RC1PWIPRDEbeEAaPTLn-00oiZnXoVouEk9AnYE4w,
|
|
|
92
92
|
kafka/producer/__init__.py,sha256=i3Wxih0NHjmqCkRNE54ial8fBp9siqabUE6ZGyL6oX8,122
|
|
93
93
|
kafka/producer/__main__.py,sha256=iOHPnGtLJTZk0Yo7Czh88CE4r3ROTDCx7IXshJwNxVw,112
|
|
94
94
|
kafka/producer/future.py,sha256=Bsdl02ZmZXbhEg6H0oamkswfOtKmsa64_2S5ceBbvOs,3120
|
|
95
|
-
kafka/producer/kafka.py,sha256=
|
|
95
|
+
kafka/producer/kafka.py,sha256=_ORqiAp5gciGlBIUx5wDOapbem7yyaR4gaP6S6iy4rk,53657
|
|
96
96
|
kafka/producer/producer_batch.py,sha256=P6Pk2D3ZsN_dCriN1_USgR6k4cTKGLXmaUBilkK7mak,7725
|
|
97
97
|
kafka/producer/record_accumulator.py,sha256=06yzb76bIEG1mRw8lJG8Pti4N06x0zL9G-USJJj77oc,22114
|
|
98
98
|
kafka/producer/sender.py,sha256=IpkH55EUg5F6U5IVRkwU_wNQRQdduwEZtFxeZlJAGnI,40944
|
|
@@ -116,13 +116,13 @@ kafka/protocol/list_offsets.py,sha256=3kvif8X-B2LBSpR3qwbkGYyJ0GLKbQdENDGpxWV0sc
|
|
|
116
116
|
kafka/protocol/message.py,sha256=9wNwJvfl9bsrdk_YcxbmAFjgvwZ5R1EBLSif2KILg9s,7657
|
|
117
117
|
kafka/protocol/metadata.py,sha256=4r5Hc-fkq4QFNh5uU_T_pk892Elhz05J1ub1KEcecRA,8831
|
|
118
118
|
kafka/protocol/offset_for_leader_epoch.py,sha256=aunp-LMIuwcCsKwvgBZ8OcUhcgb0blaq5d3PAh22JOo,4304
|
|
119
|
-
kafka/protocol/parser.py,sha256=
|
|
119
|
+
kafka/protocol/parser.py,sha256=USdmlkrURGH_mLoMksar5q5MJu-TTdfZG0HNRGwQWN4,7260
|
|
120
120
|
kafka/protocol/produce.py,sha256=4HMiSoHiW-i3ue2YsjhsbQcl9eN034rFOwIZ1RQe6-g,6540
|
|
121
121
|
kafka/protocol/sasl_authenticate.py,sha256=HaFAHPRhCKgyGEoJ5LwGffcpMUBNCphgBgXCsITLho8,1150
|
|
122
122
|
kafka/protocol/sasl_handshake.py,sha256=WzQh9HBRemXvShrczkN4rd4SM-hNdes1khMzPRvcRQQ,982
|
|
123
123
|
kafka/protocol/struct.py,sha256=iHVlM7QySnVDBbN_v4mRFmU6LJ6kslvITRdaYIsus-Q,2377
|
|
124
124
|
kafka/protocol/txn_offset_commit.py,sha256=_6Wr-SabUd9q09Tj9oG43AVZcqlW3LYbqXNW1Pvk9vs,2250
|
|
125
|
-
kafka/protocol/types.py,sha256=
|
|
125
|
+
kafka/protocol/types.py,sha256=SXAlAfrHi5q5siMxYxiMcWN_eX0fjcQs7ZRphpuUT-A,11237
|
|
126
126
|
kafka/record/__init__.py,sha256=Q20hP_R5XX3AEnAlPkpoWzTLShESvxUT2OLXmI-JYEQ,129
|
|
127
127
|
kafka/record/_crc32c.py,sha256=Ok-P62Yvg6D6rMGM9Z56OMjZWQlnps4xBbakg-sdxvI,5761
|
|
128
128
|
kafka/record/abc.py,sha256=z1UYURHbD2RyyGRpVXKP598jck5eXU9p4M6iUo6ZSFo,4110
|
|
@@ -136,7 +136,7 @@ kafka/sasl/gssapi.py,sha256=pwLxXqcmJJxkuFQUoEfX5PWgZxr-8TziuRCg9K7fO3E,4705
|
|
|
136
136
|
kafka/sasl/msk.py,sha256=FCv0uUTQKjvR2gIGyiv-dlwIvkpvEtaHvhqhXtC2q8w,8101
|
|
137
137
|
kafka/sasl/oauth.py,sha256=dh87tVi-dlS5lIzgYsC4m7IXUhlLdejaMb9Ua6oYaB0,3425
|
|
138
138
|
kafka/sasl/plain.py,sha256=PMfoWT856wx6nF_LhpfPKEnD7BRNx5l6rDhAqxBnMWU,1317
|
|
139
|
-
kafka/sasl/scram.py,sha256=
|
|
139
|
+
kafka/sasl/scram.py,sha256=Kvve3CP9QGKx4ZOFeg8aBB1683FVTWEI0l0iPthPx0E,5312
|
|
140
140
|
kafka/sasl/sspi.py,sha256=RUIVyWCEdlJPV1oj7bdzG8gORvFyR_9Bt79TzIohwMM,5001
|
|
141
141
|
kafka/serializer/__init__.py,sha256=_I4utl_8nNhcRzLLezFtwYX5akk6QKYmxa1HanRlYPU,103
|
|
142
142
|
kafka/serializer/abstract.py,sha256=doiXDkMYt2SEHRarBdd8xVZKvr5S1qPdNEtl4syWA6Q,486
|
|
@@ -145,7 +145,7 @@ kafka/vendor/enum34.py,sha256=-u-lxAiJMt6ru4Do7NUDY9OpeWkYJMksb2xengJawFE,31204
|
|
|
145
145
|
kafka/vendor/selectors34.py,sha256=gxejLO4eXf8mRSGXaQiknPig3GdX1rtsZiYOQJVuAy8,20594
|
|
146
146
|
kafka/vendor/six.py,sha256=lLBa9_HrANP5BMZ7twEzg1M3wofwPmXyptuWmHX0brY,34826
|
|
147
147
|
kafka/vendor/socketpair.py,sha256=Fi3PoY1Okkppab720wFk1BhHXyjcw7hi5DwhqrYZH2Y,2737
|
|
148
|
-
kafka_python-2.3.
|
|
149
|
-
kafka_python-2.3.
|
|
150
|
-
kafka_python-2.3.
|
|
151
|
-
kafka_python-2.3.
|
|
148
|
+
kafka_python-2.3.2.dist-info/METADATA,sha256=Ez25y7B9ri7KkPiqr8iCpJPJ7TBOqfILxj0cA259dGA,9505
|
|
149
|
+
kafka_python-2.3.2.dist-info/WHEEL,sha256=a5ogxI61vGGBHYbAAwISCXsfU7mxKr76gTpCEmSNOI8,109
|
|
150
|
+
kafka_python-2.3.2.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
|
|
151
|
+
kafka_python-2.3.2.dist-info/RECORD,,
|
|
File without changes
|