kafka-python 3.0.0__py3-none-any.whl → 3.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka/admin/client.py +2 -2
- kafka/consumer/fetcher.py +30 -0
- kafka/consumer/group.py +26 -6
- kafka/coordinator/base.py +8 -10
- kafka/net/selector.py +34 -3
- kafka/partitioner/abc.py +44 -0
- kafka/producer/future.py +38 -0
- kafka/producer/kafka.py +2 -2
- kafka/structs.py +22 -8
- kafka/version.py +1 -1
- {kafka_python-3.0.0.dist-info → kafka_python-3.0.2.dist-info}/METADATA +1 -1
- {kafka_python-3.0.0.dist-info → kafka_python-3.0.2.dist-info}/RECORD +16 -16
- {kafka_python-3.0.0.dist-info → kafka_python-3.0.2.dist-info}/WHEEL +0 -0
- {kafka_python-3.0.0.dist-info → kafka_python-3.0.2.dist-info}/entry_points.txt +0 -0
- {kafka_python-3.0.0.dist-info → kafka_python-3.0.2.dist-info}/licenses/LICENSE +0 -0
- {kafka_python-3.0.0.dist-info → kafka_python-3.0.2.dist-info}/top_level.txt +0 -0
kafka/admin/client.py
CHANGED
|
@@ -75,7 +75,7 @@ class KafkaAdminClient(
|
|
|
75
75
|
broker connection. Default: 5.
|
|
76
76
|
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
77
77
|
Used to avoid OOM when decoding malformed network message header.
|
|
78
|
-
Default:
|
|
78
|
+
Default: 100_000_000.
|
|
79
79
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
80
80
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
81
81
|
system defaults). Java client defaults to 32768.
|
|
@@ -177,7 +177,7 @@ class KafkaAdminClient(
|
|
|
177
177
|
'reconnect_backoff_ms': 50,
|
|
178
178
|
'reconnect_backoff_max_ms': 30000,
|
|
179
179
|
'max_in_flight_requests_per_connection': 5,
|
|
180
|
-
'receive_message_max_bytes':
|
|
180
|
+
'receive_message_max_bytes': 100_000_000,
|
|
181
181
|
'receive_buffer_bytes': None,
|
|
182
182
|
'send_buffer_bytes': None,
|
|
183
183
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
kafka/consumer/fetcher.py
CHANGED
|
@@ -26,6 +26,36 @@ _LOGGED_DESERIALIZE_WARNING = False
|
|
|
26
26
|
ConsumerRecord = collections.namedtuple("ConsumerRecord",
|
|
27
27
|
["topic", "partition", "leader_epoch", "offset", "timestamp", "timestamp_type",
|
|
28
28
|
"key", "value", "headers", "checksum", "serialized_key_size", "serialized_value_size", "serialized_header_size"])
|
|
29
|
+
ConsumerRecord.__doc__ = """A single record (message) consumed from a topic partition.
|
|
30
|
+
|
|
31
|
+
Yielded by :meth:`~kafka.KafkaConsumer.poll` (inside the returned
|
|
32
|
+
``{TopicPartition: [ConsumerRecord, ...]}`` mapping) and by iterating
|
|
33
|
+
over a :class:`~kafka.KafkaConsumer`. ``key`` and ``value`` are decoded
|
|
34
|
+
by the consumer's configured deserializers.
|
|
35
|
+
|
|
36
|
+
Keyword Arguments:
|
|
37
|
+
topic (str): The topic this record was received from.
|
|
38
|
+
partition (int): The partition this record was received from.
|
|
39
|
+
leader_epoch (int): The partition leader epoch for this record, or -1
|
|
40
|
+
if unknown.
|
|
41
|
+
offset (int): The position of this record in the topic partition.
|
|
42
|
+
timestamp (int): The timestamp of this record, in milliseconds since
|
|
43
|
+
the epoch (UTC), or -1 if unknown.
|
|
44
|
+
timestamp_type (int): The type of the timestamp: 0 for CreateTime (set
|
|
45
|
+
by the producer) or 1 for LogAppendTime (set by the broker).
|
|
46
|
+
key: The (deserialized) key of the record, or None.
|
|
47
|
+
value: The (deserialized) value of the record, or None.
|
|
48
|
+
headers (list): A list of ``(key, value)`` header tuples, where key is
|
|
49
|
+
a str and value is bytes.
|
|
50
|
+
checksum (int): Deprecated. The CRC32 checksum of the record, or None.
|
|
51
|
+
Removed in message format v2 (Kafka 0.11+).
|
|
52
|
+
serialized_key_size (int): The size of the serialized, uncompressed key
|
|
53
|
+
in bytes, or -1 if the key is None.
|
|
54
|
+
serialized_value_size (int): The size of the serialized, uncompressed
|
|
55
|
+
value in bytes, or -1 if the value is None.
|
|
56
|
+
serialized_header_size (int): The size of the serialized, uncompressed
|
|
57
|
+
headers in bytes, or -1 if there are no headers.
|
|
58
|
+
"""
|
|
29
59
|
|
|
30
60
|
|
|
31
61
|
CompletedFetch = collections.namedtuple("CompletedFetch",
|
kafka/consumer/group.py
CHANGED
|
@@ -180,7 +180,9 @@ class KafkaConsumer:
|
|
|
180
180
|
rebalances. Default: 3000
|
|
181
181
|
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
182
182
|
Used to avoid OOM when decoding malformed network message header.
|
|
183
|
-
|
|
183
|
+
Automatically raised to ``fetch_max_bytes + max_partition_fetch_bytes``
|
|
184
|
+
if configured lower, since a FetchResponse frame may be that large.
|
|
185
|
+
Default: 100_000_000.
|
|
184
186
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
185
187
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
186
188
|
system defaults). The java client defaults to 32768.
|
|
@@ -328,7 +330,7 @@ class KafkaConsumer:
|
|
|
328
330
|
'heartbeat_interval_ms': 3000,
|
|
329
331
|
'receive_buffer_bytes': None,
|
|
330
332
|
'send_buffer_bytes': None,
|
|
331
|
-
'receive_message_max_bytes':
|
|
333
|
+
'receive_message_max_bytes': 100_000_000,
|
|
332
334
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
|
333
335
|
'consumer_timeout_ms': float('inf'),
|
|
334
336
|
'security_protocol': 'PLAINTEXT',
|
|
@@ -389,6 +391,20 @@ class KafkaConsumer:
|
|
|
389
391
|
"fetch_max_wait_ms ({})."
|
|
390
392
|
.format(connections_max_idle_ms, request_timeout_ms, fetch_max_wait_ms))
|
|
391
393
|
|
|
394
|
+
# fetch_max_bytes (KIP-74) is a soft cap the broker applies to the
|
|
395
|
+
# *record data* in a FetchResponse body, not to the whole frame. The
|
|
396
|
+
# frame also carries the response header + per-topic/per-partition
|
|
397
|
+
# metadata, and the broker returns at least the first record batch even
|
|
398
|
+
# when it alone exceeds fetch_max_bytes. So a FetchResponse frame can be
|
|
399
|
+
# somewhat larger than fetch_max_bytes; give the frame-size guard
|
|
400
|
+
# max_partition_fetch_bytes of headroom over it so legitimate responses
|
|
401
|
+
# aren't rejected as InvalidReceiveError. Auto-raise so the guard tracks
|
|
402
|
+
# the fetch sizing unless the user explicitly configured a larger value.
|
|
403
|
+
fetch_response_max_bytes = (
|
|
404
|
+
self.config['fetch_max_bytes'] + self.config['max_partition_fetch_bytes'])
|
|
405
|
+
if self.config['receive_message_max_bytes'] < fetch_response_max_bytes:
|
|
406
|
+
self.config['receive_message_max_bytes'] = fetch_response_max_bytes
|
|
407
|
+
|
|
392
408
|
if self.config['metrics_enabled']:
|
|
393
409
|
metrics_tags = {'client-id': self.config['client_id']}
|
|
394
410
|
metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
|
|
@@ -662,13 +678,17 @@ class KafkaConsumer:
|
|
|
662
678
|
self._coordinator.commit_offsets_sync(offsets, timeout_ms=timeout_ms)
|
|
663
679
|
|
|
664
680
|
def group_metadata(self):
|
|
665
|
-
"""Return a snapshot of this consumer's group membership
|
|
681
|
+
"""Return a snapshot of this consumer's group membership.
|
|
666
682
|
|
|
667
683
|
Pass the result to KafkaProducer.send_offsets_to_transaction() so the
|
|
668
684
|
broker can fence stale instances of this consumer when committing
|
|
669
|
-
offsets inside a transaction. The snapshot
|
|
670
|
-
|
|
671
|
-
|
|
685
|
+
offsets inside a transaction (KIP-447). The snapshot also exposes the
|
|
686
|
+
current MemberState (``state``), so callers can observe whether the
|
|
687
|
+
consumer has converged on a stable assignment.
|
|
688
|
+
|
|
689
|
+
The snapshot is always safe to call: if no group_id is configured
|
|
690
|
+
(manual assignment) the returned ConsumerGroupMetadata has
|
|
691
|
+
group_id=None and is permanently unjoined.
|
|
672
692
|
|
|
673
693
|
Returns:
|
|
674
694
|
ConsumerGroupMetadata
|
kafka/coordinator/base.py
CHANGED
|
@@ -16,19 +16,13 @@ from kafka.protocol.consumer import (
|
|
|
16
16
|
HeartbeatRequest, JoinGroupRequest, LeaveGroupRequest, SyncGroupRequest,
|
|
17
17
|
DEFAULT_GENERATION_ID, UNKNOWN_MEMBER_ID,
|
|
18
18
|
)
|
|
19
|
-
from kafka.structs import ConsumerGroupMetadata
|
|
19
|
+
from kafka.structs import ConsumerGroupMetadata, MemberState
|
|
20
20
|
from kafka.util import Timer
|
|
21
21
|
|
|
22
22
|
log = logging.getLogger('kafka.coordinator')
|
|
23
23
|
heartbeat_log = logging.getLogger('kafka.coordinator.heartbeat')
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
class MemberState:
|
|
27
|
-
UNJOINED = '<unjoined>' # the client is not part of a group
|
|
28
|
-
REBALANCING = '<rebalancing>' # the client has begun rebalancing
|
|
29
|
-
STABLE = '<stable>' # the client has joined and is sending heartbeats
|
|
30
|
-
|
|
31
|
-
|
|
32
26
|
class Generation:
|
|
33
27
|
def __init__(self, generation_id, member_id, protocol):
|
|
34
28
|
self.generation_id = generation_id
|
|
@@ -891,13 +885,16 @@ class BaseCoordinator(ABC):
|
|
|
891
885
|
return self._generation
|
|
892
886
|
|
|
893
887
|
def group_metadata(self):
|
|
894
|
-
"""Return a snapshot of this member's group
|
|
888
|
+
"""Return a snapshot of this member's group membership.
|
|
895
889
|
|
|
896
890
|
Returns the current generation_id / member_id / group_instance_id even
|
|
897
891
|
when the group is not stable; the caller (typically
|
|
898
892
|
KafkaProducer.send_offsets_to_transaction) needs whatever is current
|
|
899
|
-
so the broker can fence stale instances. If the consumer has
|
|
900
|
-
joined, the snapshot has the no-generation defaults.
|
|
893
|
+
so the broker can fence stale instances (KIP-447). If the consumer has
|
|
894
|
+
never joined, the snapshot has the no-generation defaults.
|
|
895
|
+
|
|
896
|
+
Also carries the live MemberState (``state``) so callers can observe
|
|
897
|
+
whether the group has converged (it is ignored by the fencing path).
|
|
901
898
|
"""
|
|
902
899
|
with self._lock:
|
|
903
900
|
return ConsumerGroupMetadata(
|
|
@@ -905,6 +902,7 @@ class BaseCoordinator(ABC):
|
|
|
905
902
|
generation_id=self._generation.generation_id,
|
|
906
903
|
member_id=self._generation.member_id,
|
|
907
904
|
group_instance_id=self.group_instance_id,
|
|
905
|
+
state=self.state,
|
|
908
906
|
)
|
|
909
907
|
|
|
910
908
|
# deprecated
|
kafka/net/selector.py
CHANGED
|
@@ -122,6 +122,21 @@ class Task:
|
|
|
122
122
|
raise RuntimeError('Task exception is already set!')
|
|
123
123
|
self._exc = exc
|
|
124
124
|
|
|
125
|
+
def close(self):
|
|
126
|
+
stack = self._stack
|
|
127
|
+
while stack:
|
|
128
|
+
coro, stack = stack
|
|
129
|
+
if inspect.isgenerator(coro) or inspect.iscoroutine(coro):
|
|
130
|
+
try:
|
|
131
|
+
coro.close()
|
|
132
|
+
except ValueError:
|
|
133
|
+
# currently-executing coroutine -- can't close it from
|
|
134
|
+
# within itself; bail without corrupting _stack.
|
|
135
|
+
return
|
|
136
|
+
except Exception:
|
|
137
|
+
log.exception('Error closing coroutine for cancelled task')
|
|
138
|
+
self._stack = None
|
|
139
|
+
|
|
125
140
|
@property
|
|
126
141
|
def is_done(self):
|
|
127
142
|
return self._stack is None
|
|
@@ -363,13 +378,29 @@ class NetworkSelector:
|
|
|
363
378
|
result = await result
|
|
364
379
|
return result
|
|
365
380
|
|
|
366
|
-
def
|
|
381
|
+
def _remove_scheduled(self, task):
|
|
367
382
|
if task.scheduled_at is not None:
|
|
368
|
-
|
|
383
|
+
try:
|
|
384
|
+
self._scheduled.remove((task.scheduled_at, task))
|
|
385
|
+
except ValueError:
|
|
386
|
+
pass
|
|
387
|
+
else:
|
|
388
|
+
# re-heapify to ensure heap structure is valid
|
|
389
|
+
heapq.heapify(self._scheduled)
|
|
369
390
|
task.scheduled_at = None
|
|
370
391
|
|
|
392
|
+
def _retire_task(self, task):
|
|
393
|
+
if task is self._current:
|
|
394
|
+
return
|
|
395
|
+
self._pending_tasks.discard(task)
|
|
396
|
+
task.close()
|
|
397
|
+
|
|
398
|
+
def unschedule(self, task):
|
|
399
|
+
self._remove_scheduled(task)
|
|
400
|
+
self._retire_task(task)
|
|
401
|
+
|
|
371
402
|
def reschedule(self, when, task):
|
|
372
|
-
self.
|
|
403
|
+
self._remove_scheduled(task)
|
|
373
404
|
self.call_at(when, task)
|
|
374
405
|
return task
|
|
375
406
|
|
kafka/partitioner/abc.py
CHANGED
|
@@ -2,7 +2,51 @@ from abc import ABC, abstractmethod
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class Partitioner(ABC):
|
|
5
|
+
"""Base class for pluggable partition selection strategies.
|
|
6
|
+
|
|
7
|
+
A :class:`~kafka.KafkaProducer` consults its configured partitioner to
|
|
8
|
+
choose a partition for each record whose ``partition`` was not supplied
|
|
9
|
+
explicitly to :meth:`~kafka.KafkaProducer.send`. Subclass this and
|
|
10
|
+
implement :meth:`partition`, then pass an instance via the
|
|
11
|
+
``partitioner`` config argument.
|
|
12
|
+
|
|
13
|
+
Two built-in implementations are provided: ``DefaultPartitioner``
|
|
14
|
+
(murmur2-hash keyed records, random partition for null keys) and
|
|
15
|
+
``StickyPartitioner`` (KIP-480; null-key records stick to one partition
|
|
16
|
+
per topic until a new batch is started, then rotate).
|
|
17
|
+
|
|
18
|
+
Sticky-style partitioners may additionally implement an optional
|
|
19
|
+
``on_new_batch(self, topic, cluster, prev_partition)`` hook, which the
|
|
20
|
+
producer calls when a null-key record would have triggered a fresh
|
|
21
|
+
batch, giving the partitioner a chance to rotate off its current sticky
|
|
22
|
+
choice. The hook is looked up with ``getattr``, so it is entirely
|
|
23
|
+
optional.
|
|
24
|
+
"""
|
|
25
|
+
|
|
5
26
|
@abstractmethod
|
|
6
27
|
def partition(self, topic, key, serialized_key, value, serialized_value, cluster):
|
|
28
|
+
"""Choose a partition for a record.
|
|
29
|
+
|
|
30
|
+
Arguments:
|
|
31
|
+
topic (str): The topic the record is destined for.
|
|
32
|
+
key: The user-supplied key, before serialization. May be None.
|
|
33
|
+
serialized_key (bytes): The post-serializer key bytes, or None
|
|
34
|
+
when the caller passed ``key=None``.
|
|
35
|
+
value: The user-supplied value, before serialization.
|
|
36
|
+
serialized_value (bytes): The post-serializer value bytes, or
|
|
37
|
+
None when the caller passed ``value=None``.
|
|
38
|
+
cluster (ClusterMetadata): A live cluster snapshot. Use
|
|
39
|
+
``cluster.partitions_for_topic(topic)`` for all partitions
|
|
40
|
+
and ``cluster.available_partitions_for_topic(topic)`` for
|
|
41
|
+
those whose leader is currently known.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
int: The partition id to assign the record to.
|
|
45
|
+
|
|
46
|
+
Raises:
|
|
47
|
+
ValueError: If the topic is not present in cluster metadata.
|
|
48
|
+
Partitioner exceptions surface to the caller via the
|
|
49
|
+
returned :class:`~kafka.producer.future.FutureRecordMetadata`.
|
|
50
|
+
"""
|
|
7
51
|
pass
|
|
8
52
|
|
kafka/producer/future.py
CHANGED
|
@@ -30,6 +30,19 @@ class FutureProduceResult(Future):
|
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
class FutureRecordMetadata(Future):
|
|
33
|
+
"""An asynchronous handle to the result of a single :meth:`~kafka.KafkaProducer.send`.
|
|
34
|
+
|
|
35
|
+
:meth:`~kafka.KafkaProducer.send` returns one of these immediately,
|
|
36
|
+
before the record has been transmitted to the broker. Call :meth:`get`
|
|
37
|
+
to block until the record is acknowledged and obtain its
|
|
38
|
+
:class:`RecordMetadata`, or register callbacks via
|
|
39
|
+
:meth:`~kafka.future.Future.add_callback` /
|
|
40
|
+
:meth:`~kafka.future.Future.add_errback` to be notified without
|
|
41
|
+
blocking. The future resolves successfully once the containing batch is
|
|
42
|
+
acknowledged according to the producer's ``acks`` configuration, or
|
|
43
|
+
fails with the relevant exception (for example
|
|
44
|
+
:class:`~kafka.errors.KafkaTimeoutError`).
|
|
45
|
+
"""
|
|
33
46
|
__slots__ = ('_produce_future', 'args')
|
|
34
47
|
def __init__(self, produce_future, batch_index, timestamp_ms, checksum, serialized_key_size, serialized_value_size, serialized_header_size):
|
|
35
48
|
super().__init__()
|
|
@@ -99,3 +112,28 @@ class FutureRecordMetadata(Future):
|
|
|
99
112
|
RecordMetadata = collections.namedtuple(
|
|
100
113
|
'RecordMetadata', ['topic', 'partition', 'topic_partition', 'offset', 'timestamp',
|
|
101
114
|
'checksum', 'serialized_key_size', 'serialized_value_size', 'serialized_header_size'])
|
|
115
|
+
RecordMetadata.__doc__ = """Metadata about a record that has been acknowledged by the broker.
|
|
116
|
+
|
|
117
|
+
Returned by :meth:`FutureRecordMetadata.get`, which resolves once the
|
|
118
|
+
batch containing the record has been acknowledged according to the
|
|
119
|
+
producer's ``acks`` configuration.
|
|
120
|
+
|
|
121
|
+
Keyword Arguments:
|
|
122
|
+
topic (str): The topic the record was appended to.
|
|
123
|
+
partition (int): The partition the record was appended to.
|
|
124
|
+
topic_partition (TopicPartition): The ``(topic, partition)`` the record
|
|
125
|
+
was appended to.
|
|
126
|
+
offset (int): The offset of the record in the topic partition, or -1 if
|
|
127
|
+
the broker did not assign one (e.g. ``acks=0``).
|
|
128
|
+
timestamp (int): The timestamp of the record, in milliseconds since the
|
|
129
|
+
epoch (UTC). For CreateTime this is the producer-supplied timestamp;
|
|
130
|
+
for LogAppendTime it is the broker-assigned timestamp.
|
|
131
|
+
checksum (int): Deprecated. The CRC32 checksum of the record, or None.
|
|
132
|
+
Removed in message format v2 (Kafka 0.11+).
|
|
133
|
+
serialized_key_size (int): The size of the serialized, uncompressed key
|
|
134
|
+
in bytes, or -1 if the key is None.
|
|
135
|
+
serialized_value_size (int): The size of the serialized, uncompressed
|
|
136
|
+
value in bytes, or -1 if the value is None.
|
|
137
|
+
serialized_header_size (int): The size of the serialized, uncompressed
|
|
138
|
+
headers in bytes, or -1 if there are no headers.
|
|
139
|
+
"""
|
kafka/producer/kafka.py
CHANGED
|
@@ -282,7 +282,7 @@ class KafkaProducer:
|
|
|
282
282
|
Default: 30000.
|
|
283
283
|
receive_message_max_bytes (int): Maximum allowed network frame size.
|
|
284
284
|
Used to avoid OOM when decoding malformed network message header.
|
|
285
|
-
Default:
|
|
285
|
+
Default: 100_000_000.
|
|
286
286
|
receive_buffer_bytes (int): The size of the TCP receive buffer
|
|
287
287
|
(SO_RCVBUF) to use when reading data. Default: None (relies on
|
|
288
288
|
system defaults). Java client defaults to 32768.
|
|
@@ -425,7 +425,7 @@ class KafkaProducer:
|
|
|
425
425
|
'client_dns_lookup': 'use_all_dns_ips',
|
|
426
426
|
'retry_backoff_ms': 100,
|
|
427
427
|
'request_timeout_ms': 30000,
|
|
428
|
-
'receive_message_max_bytes':
|
|
428
|
+
'receive_message_max_bytes': 100_000_000,
|
|
429
429
|
'receive_buffer_bytes': None,
|
|
430
430
|
'send_buffer_bytes': None,
|
|
431
431
|
'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
|
kafka/structs.py
CHANGED
|
@@ -51,19 +51,33 @@ Keyword Arguments:
|
|
|
51
51
|
"""
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
class MemberState:
|
|
55
|
+
UNJOINED = '<unjoined>' # the client is not part of a group
|
|
56
|
+
REBALANCING = '<rebalancing>' # the client has begun rebalancing
|
|
57
|
+
STABLE = '<stable>' # the client has joined and is sending heartbeats
|
|
58
|
+
|
|
59
|
+
|
|
54
60
|
ConsumerGroupMetadata = namedtuple("ConsumerGroupMetadata",
|
|
55
|
-
["group_id", "generation_id", "member_id", "group_instance_id"],
|
|
56
|
-
defaults=[-1, '', None])
|
|
57
|
-
ConsumerGroupMetadata.__doc__ = """A snapshot of a consumer's group membership
|
|
61
|
+
["group_id", "generation_id", "member_id", "group_instance_id", "state"],
|
|
62
|
+
defaults=[None, -1, '', None, MemberState.UNJOINED])
|
|
63
|
+
ConsumerGroupMetadata.__doc__ = """A snapshot of a consumer's group membership.
|
|
64
|
+
|
|
65
|
+
The first four fields are the KIP-447 fencing identity: pass the snapshot to
|
|
66
|
+
KafkaProducer.send_offsets_to_transaction() so the broker can fence stale
|
|
67
|
+
consumer instances when committing offsets inside a transaction. The broker
|
|
68
|
+
uses member_id + generation_id + group_instance_id to verify the producer is
|
|
69
|
+
acting on behalf of the current group generation.
|
|
58
70
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
71
|
+
The ``state`` field exposes the live MemberState (it is ignored by the
|
|
72
|
+
producer/fencing path). It lets callers observe whether the consumer has
|
|
73
|
+
converged on a stable assignment - useful for monitoring and for tests that
|
|
74
|
+
wait for a group to finish rebalancing.
|
|
63
75
|
|
|
64
76
|
Keyword Arguments:
|
|
65
|
-
group_id (str): The consumer group id.
|
|
77
|
+
group_id (str): The consumer group id, or None for manual assignment.
|
|
66
78
|
generation_id (int): The current generation id (-1 if unjoined).
|
|
67
79
|
member_id (str): The current member id ('' if unjoined).
|
|
68
80
|
group_instance_id (str): The static membership instance id, or None.
|
|
81
|
+
state (str): The current MemberState (one of MemberState.UNJOINED,
|
|
82
|
+
MemberState.REBALANCING, MemberState.STABLE).
|
|
69
83
|
"""
|
kafka/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = '3.0.
|
|
1
|
+
__version__ = '3.0.2'
|
|
@@ -4,9 +4,9 @@ kafka/cluster.py,sha256=P1-ylBAa33FAelwufsod_ouIp2FCfE-MI6_sNnxJyJc,32429
|
|
|
4
4
|
kafka/codec.py,sha256=xApRCUaj6HsGM6T5q-AeK9kRPkH6Au1wqsXwXhwE2Lo,9843
|
|
5
5
|
kafka/errors.py,sha256=9owgtTIcttdTCD1vAF9wz5jL9G72ljpR7M-vjLKEGvY,33206
|
|
6
6
|
kafka/future.py,sha256=INmMIy2-BPxJ4zlw-V_i8Tj_OG9DMbyYyUEjOQ5pn_o,5440
|
|
7
|
-
kafka/structs.py,sha256=
|
|
7
|
+
kafka/structs.py,sha256=ZOFmVJRbhSXykkw9cnuv-DGYE7g2KjaUGsytSOv4xVw,3146
|
|
8
8
|
kafka/util.py,sha256=hvSdRbh5-IsUbpD_BqjabjEldtNmXufZH68b_xCkNjE,5194
|
|
9
|
-
kafka/version.py,sha256=
|
|
9
|
+
kafka/version.py,sha256=FDUC8xOuizaRKSkyG1VOmlDwzcPJgbzFUWz58fhdUGQ,22
|
|
10
10
|
kafka/admin/__init__.py,sha256=nTXg2CQo8Bydxsj80pVBzF2-jq4IhK7_gLnboeNWG60,1575
|
|
11
11
|
kafka/admin/__main__.py,sha256=l5ujkHa3lBFEbKPCl7U0_pi6IIgj1IJfmB219vtcj50,69
|
|
12
12
|
kafka/admin/_acls.py,sha256=QJlHguEM9t46qlABoQqTEDFQErDgG2RKwfV1NOKeKmU,13983
|
|
@@ -17,7 +17,7 @@ kafka/admin/_partitions.py,sha256=xQ5toaocldfZOnghfa2-q95J7AdJjDJNMggzqCjRy9Q,26
|
|
|
17
17
|
kafka/admin/_topics.py,sha256=B9xjKQXHcFosOCBg1faSvhrbh7AoqalUkn4t28jxxAo,12429
|
|
18
18
|
kafka/admin/_transactions.py,sha256=D4iLBEu3aXbykFDm6Q1knVXc9GUMuap6TW4RI1U2Rvs,19917
|
|
19
19
|
kafka/admin/_users.py,sha256=NKaxwUuskMTyRHXU__ImU7IGH1iaqrPe2Zp36Pvso54,6909
|
|
20
|
-
kafka/admin/client.py,sha256=
|
|
20
|
+
kafka/admin/client.py,sha256=YDitvRX6y1B7tz2mExzPtWKFJyrTOF5nxrqh-kpStW4,18316
|
|
21
21
|
kafka/benchmarks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
kafka/benchmarks/consumer_performance.py,sha256=HBJTS7Szr6y3OAszrTHskr-5snW5ia8tk16ZtSz2gyA,4811
|
|
23
23
|
kafka/benchmarks/load_example.py,sha256=aivu9HHYtkODnecIx4AxeLUx9U42bRjGwtWa_b_MO44,3425
|
|
@@ -83,11 +83,11 @@ kafka/cli/consumer/__init__.py,sha256=eNb-AFzLj9QJrEeZZsp427uZ-BDGSIG9Bp3bbOYgvp
|
|
|
83
83
|
kafka/cli/producer/__init__.py,sha256=W3Rs0Darw2f3h1IZZawep1K-LVg2AA0bEuwEEaqPjKg,1682
|
|
84
84
|
kafka/consumer/__init__.py,sha256=I7nuPUb103S1CvCqcudbIRG_isJbRgTXkK1mpHIhoVs,82
|
|
85
85
|
kafka/consumer/__main__.py,sha256=bnxM30LbGVvLb0ZBcgJEt_8bZHADJpboHZCxzd8QEhA,72
|
|
86
|
-
kafka/consumer/fetcher.py,sha256=
|
|
87
|
-
kafka/consumer/group.py,sha256=
|
|
86
|
+
kafka/consumer/fetcher.py,sha256=hOj0ecZxsp83pUf6K0PZY01PCpEZhBjoJymk9bpv4Qc,102423
|
|
87
|
+
kafka/consumer/group.py,sha256=lJo2U_wHSk_j-qwfa2GeU_gOZq5kWbxh2qAkjyMer6Q,68423
|
|
88
88
|
kafka/consumer/subscription_state.py,sha256=H1bDpDdy3uk3U8TtyjCuo_HmaZmSpOYevaYtrc7LvgY,38567
|
|
89
89
|
kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
90
|
-
kafka/coordinator/base.py,sha256=
|
|
90
|
+
kafka/coordinator/base.py,sha256=gIzjv60NIC6zlkgUm1w-yG-S5i6fy_pSo3BJEBbOD9M,57841
|
|
91
91
|
kafka/coordinator/consumer.py,sha256=cQ-zo0p_lL5H3j8DT4Q6ZrBBxCWWh4p1ub1X2IHlx5w,60186
|
|
92
92
|
kafka/coordinator/heartbeat.py,sha256=JUlq_-TqOd6kwiFcapa20sgaM8lSZ2JgObEzeTTcDm0,3229
|
|
93
93
|
kafka/coordinator/subscription.py,sha256=rq-syb2zwjRiY1797Yi4dxTZH-QFJSsaNAAJVwBHrqA,873
|
|
@@ -133,7 +133,7 @@ kafka/net/http_connect.py,sha256=j8R1yLNwhTF-nQVMEgRF88mOF5-v1Sdj7uwkBvvg5PU,513
|
|
|
133
133
|
kafka/net/inet.py,sha256=zMo0NTCS7EY81uoNPri5ZHKcCnRVUIcCJiW6BeZWnCs,4730
|
|
134
134
|
kafka/net/manager.py,sha256=nq7eKgzkazQ3sFLXQ-IOIFQ3FdyZYzfKxYxVrwYPGsc,19427
|
|
135
135
|
kafka/net/metrics.py,sha256=-cdWXQtonCO7-LpovLoEcWM-_3R9sAtyQxWf_QHOAWE,7299
|
|
136
|
-
kafka/net/selector.py,sha256=
|
|
136
|
+
kafka/net/selector.py,sha256=4_g3AtYQfmhMjzFgtFOE_TbkIIifFWW5B8zvQmyBJCI,25733
|
|
137
137
|
kafka/net/socks5.py,sha256=euTwJl2avRmCJhVKyGmhZWInqgY6MNHPA1Tz2f2J1Z0,10361
|
|
138
138
|
kafka/net/transport.py,sha256=hfdoQ9dMOXrMxxLJ6c4A69OGN3_FuRvDL5UuFJ1X-sA,15116
|
|
139
139
|
kafka/net/wakeup_notifier.py,sha256=hmxwrrqZ8zFHSHGYaorV7vdOoKo9SF3-VoS7YGkPq_w,2926
|
|
@@ -146,13 +146,13 @@ kafka/net/sasl/plain.py,sha256=Vx3VfF2CRzHZV8A--1VRRyhQcJqadgaCmd9Gb2dpfes,1402
|
|
|
146
146
|
kafka/net/sasl/scram.py,sha256=Q87VPsAoH-ujtAHOhWzvqY5Kks6BJ7NQPOXZ5-Fwbz0,5272
|
|
147
147
|
kafka/net/sasl/sspi.py,sha256=mIZSXWutKKo4BXMa1Cuw4uL-r8VzicrVkkiJDMhBdDA,5027
|
|
148
148
|
kafka/partitioner/__init__.py,sha256=xEF0Uta4wS0VLsOHsOZlkpnB095pavtgMBt1QVJuE_Q,204
|
|
149
|
-
kafka/partitioner/abc.py,sha256=
|
|
149
|
+
kafka/partitioner/abc.py,sha256=OXXFEvRf1sqAsebA2mKSWJ6jdKIyNgqoWTIj84me_78,2358
|
|
150
150
|
kafka/partitioner/default.py,sha256=ZD8SeTffJCPsNjo4GZ7y4cWVW6iG6NybRZuWbul231g,2635
|
|
151
151
|
kafka/partitioner/sticky.py,sha256=A3pfCrFl-jhmpqTANlRcQ5ixd_e-F3K8-d42ugaE378,4804
|
|
152
152
|
kafka/producer/__init__.py,sha256=OPWP89JbP1dCYG4HSWl1iz6KLun0oJDYfqLAr0vAfHQ,82
|
|
153
153
|
kafka/producer/__main__.py,sha256=h5u6-EfA7bJFszn0x-QV14pZt4MOJw0jiPHKuMSkCOY,72
|
|
154
|
-
kafka/producer/future.py,sha256=
|
|
155
|
-
kafka/producer/kafka.py,sha256=
|
|
154
|
+
kafka/producer/future.py,sha256=D5_8Hz-8CqJZYEVMieUk7wBhKS83skFnisGlsyosIbk,6669
|
|
155
|
+
kafka/producer/kafka.py,sha256=j01YfmZn3sNigHBNITd5CLeohu4WF0FVTFE1E6Pj5O8,59338
|
|
156
156
|
kafka/producer/producer_batch.py,sha256=gFuJNaUM9gHs0KpgQ0KFZCIlBy43OnrRyzo2xb0HSTo,8027
|
|
157
157
|
kafka/producer/record_accumulator.py,sha256=B6C64fNzD7MIFsWV-kJFrahKRVS5dYniI3OFhnUqvRw,28039
|
|
158
158
|
kafka/producer/sender.py,sha256=aV4skH8T1xncOidyS8do0Gv2jcr97u3CHx5wXlrPTik,44608
|
|
@@ -365,9 +365,9 @@ kafka/serializer/default.py,sha256=ZKzTWlG9N4vS3QXovFLygNVrnjAcMZKS0RE4OvQ6DL4,4
|
|
|
365
365
|
kafka/serializer/json.py,sha256=lErgU66KZGf33hofwlObYIaxLDk9gcolhy_rlcOuf8Y,469
|
|
366
366
|
kafka/serializer/wrapper.py,sha256=RGXFj-PXQyL3rdMXI9ACaohRa6Kg5SHOZTRruU9jluM,485
|
|
367
367
|
kafka/vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
368
|
-
kafka_python-3.0.
|
|
369
|
-
kafka_python-3.0.
|
|
370
|
-
kafka_python-3.0.
|
|
371
|
-
kafka_python-3.0.
|
|
372
|
-
kafka_python-3.0.
|
|
373
|
-
kafka_python-3.0.
|
|
368
|
+
kafka_python-3.0.2.dist-info/licenses/LICENSE,sha256=vxnoJsqm6bKl3ZWdI1Q7Ikw_k9cOvO3vcvZNsY_1fP8,11374
|
|
369
|
+
kafka_python-3.0.2.dist-info/METADATA,sha256=jQB9VNg9SS9r4_i52w7nofQF2k0ZWblwKCsiGc86a8k,11590
|
|
370
|
+
kafka_python-3.0.2.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
371
|
+
kafka_python-3.0.2.dist-info/entry_points.txt,sha256=LQvqZj3uM785ainO5HrXmukbjSrK-oPqrESqpvoR-As,51
|
|
372
|
+
kafka_python-3.0.2.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
|
|
373
|
+
kafka_python-3.0.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|