kafka-python 2.2.2__tar.gz → 2.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kafka_python-2.2.2 → kafka_python-2.2.4}/CHANGES.md +18 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/PKG-INFO +1 -1
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/client_async.py +7 -11
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/cluster.py +24 -1
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/conn.py +1 -27
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/fetcher.py +52 -36
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/group.py +17 -19
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/base.py +42 -20
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/consumer.py +72 -39
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/errors.py +12 -12
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/kafka.py +12 -24
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/util.py +39 -19
- kafka_python-2.2.4/kafka/version.py +1 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/PKG-INFO +1 -1
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_cluster.py +60 -1
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_conn.py +1 -49
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_fetcher.py +20 -10
- kafka_python-2.2.2/kafka/version.py +0 -1
- {kafka_python-2.2.2 → kafka_python-2.2.4}/AUTHORS.md +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/LICENSE +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/MANIFEST.in +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/README.rst +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/acl_resource.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/client.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/config_resource.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/new_partitions.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/new_topic.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/consumer_performance.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/load_example.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/producer_performance.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/record_batch_compose.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/record_batch_read.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/varint_speed.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/codec.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/subscription_state.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/abstract.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/range.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/roundrobin.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/partition_movements.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/sorted_set.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/sticky_assignor.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/heartbeat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/protocol.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/future.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/compound_stat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/dict_reporter.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/kafka_metric.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/measurable.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/measurable_stat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metric_config.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metric_name.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metrics.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metrics_reporter.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/quota.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/avg.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/count.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/histogram.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/max_stat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/min_stat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/percentile.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/percentiles.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/rate.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/sampled_stat.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/sensor.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/total.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/partitioner/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/partitioner/default.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/future.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/record_accumulator.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/sender.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/transaction_manager.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/abstract.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/add_offsets_to_txn.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/add_partitions_to_txn.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/admin.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/api.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/api_versions.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/broker_api_versions.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/commit.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/end_txn.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/fetch.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/find_coordinator.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/frame.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/group.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/init_producer_id.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/list_offsets.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/message.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/metadata.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/offset_for_leader_epoch.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/parser.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/pickle.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/produce.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/sasl_authenticate.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/sasl_handshake.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/struct.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/txn_offset_commit.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/types.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/_crc32c.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/abc.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/default_records.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/legacy_records.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/memory_records.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/util.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/abc.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/gssapi.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/msk.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/oauth.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/plain.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/scram.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/sspi.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/serializer/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/serializer/abstract.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/socks5_wrapper.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/structs.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/enum34.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/selectors34.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/six.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/socketpair.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/SOURCES.txt +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/dependency_links.txt +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/requires.txt +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/top_level.txt +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/pyproject.toml +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/setup.cfg +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/setup.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/__init__.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/conftest.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/fixtures.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_admin_integration.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_consumer_group.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_consumer_integration.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_producer_integration.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_sasl_integration.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_acl_comparisons.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_admin.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_api_object_implementation.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_assignors.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_client_async.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_codec.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_consumer.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_coordinator.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_metrics.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_object_conversion.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_package.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_partition_movements.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_partitioner.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_producer.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_protocol.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_record_accumulator.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_sender.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_subscription_state.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_util.py +0 -0
- {kafka_python-2.2.2 → kafka_python-2.2.4}/test/testutil.py +0 -0
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
# 2.2.4 (May 3, 2025)
|
|
2
|
+
|
|
3
|
+
Fixes
|
|
4
|
+
* Do not `reset_generation` after RebalanceInProgressError; improve CommitFailed error messages (#2614)
|
|
5
|
+
* Fix KafkaConsumer.poll() with zero timeout (#2613)
|
|
6
|
+
* Fix Fetch._reset_offsets_async() KeyError when fetching from multiple nodes (#2612)
|
|
7
|
+
|
|
8
|
+
# 2.2.3 (May 1, 2025)
|
|
9
|
+
|
|
10
|
+
Fixes
|
|
11
|
+
* Ignore leading SECURITY_PROTOCOL:// in bootstrap_servers (#2608)
|
|
12
|
+
* Only create fetch requests for ready nodes (#2607)
|
|
13
|
+
|
|
1
14
|
# 2.2.2 (Apr 30, 2025)
|
|
2
15
|
|
|
3
16
|
Fixes
|
|
@@ -58,6 +71,11 @@ Tests
|
|
|
58
71
|
* py2 test fixups
|
|
59
72
|
* Drop unused KafkaClient import from `test_fetcher`
|
|
60
73
|
|
|
74
|
+
# 2.1.6 (May 2, 2025)
|
|
75
|
+
|
|
76
|
+
Fixes
|
|
77
|
+
* Only create fetch requests for ready nodes (#2607)
|
|
78
|
+
|
|
61
79
|
# 2.1.5 (Apr 4, 2025)
|
|
62
80
|
|
|
63
81
|
Fixes
|
|
@@ -27,7 +27,7 @@ from kafka.metrics.stats import Avg, Count, Rate
|
|
|
27
27
|
from kafka.metrics.stats.rate import TimeUnit
|
|
28
28
|
from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS
|
|
29
29
|
from kafka.protocol.metadata import MetadataRequest
|
|
30
|
-
from kafka.util import Dict, WeakMethod, ensure_valid_topic_name
|
|
30
|
+
from kafka.util import Dict, Timer, WeakMethod, ensure_valid_topic_name
|
|
31
31
|
# Although this looks unused, it actually monkey-patches socket.socketpair()
|
|
32
32
|
# and should be left in as long as we're using socket.socketpair() in this file
|
|
33
33
|
from kafka.vendor import socketpair # noqa: F401
|
|
@@ -645,12 +645,8 @@ class KafkaClient(object):
|
|
|
645
645
|
"""
|
|
646
646
|
if not isinstance(timeout_ms, (int, float, type(None))):
|
|
647
647
|
raise TypeError('Invalid type for timeout: %s' % type(timeout_ms))
|
|
648
|
+
timer = Timer(timeout_ms)
|
|
648
649
|
|
|
649
|
-
begin = time.time()
|
|
650
|
-
if timeout_ms is not None:
|
|
651
|
-
timeout_at = begin + (timeout_ms / 1000)
|
|
652
|
-
else:
|
|
653
|
-
timeout_at = begin + (self.config['request_timeout_ms'] / 1000)
|
|
654
650
|
# Loop for futures, break after first loop if None
|
|
655
651
|
responses = []
|
|
656
652
|
while True:
|
|
@@ -675,7 +671,7 @@ class KafkaClient(object):
|
|
|
675
671
|
if future is not None and future.is_done:
|
|
676
672
|
timeout = 0
|
|
677
673
|
else:
|
|
678
|
-
user_timeout_ms =
|
|
674
|
+
user_timeout_ms = timer.timeout_ms if timeout_ms is not None else self.config['request_timeout_ms']
|
|
679
675
|
idle_connection_timeout_ms = self._idle_expiry_manager.next_check_ms()
|
|
680
676
|
request_timeout_ms = self._next_ifr_request_timeout_ms()
|
|
681
677
|
log.debug("Timeouts: user %f, metadata %f, idle connection %f, request %f", user_timeout_ms, metadata_timeout_ms, idle_connection_timeout_ms, request_timeout_ms)
|
|
@@ -698,7 +694,7 @@ class KafkaClient(object):
|
|
|
698
694
|
break
|
|
699
695
|
elif future.is_done:
|
|
700
696
|
break
|
|
701
|
-
elif timeout_ms is not None and
|
|
697
|
+
elif timeout_ms is not None and timer.expired:
|
|
702
698
|
break
|
|
703
699
|
|
|
704
700
|
return responses
|
|
@@ -1175,16 +1171,16 @@ class KafkaClient(object):
|
|
|
1175
1171
|
This method is useful for implementing blocking behaviour on top of the non-blocking `NetworkClient`, use it with
|
|
1176
1172
|
care.
|
|
1177
1173
|
"""
|
|
1178
|
-
|
|
1174
|
+
timer = Timer(timeout_ms)
|
|
1179
1175
|
self.poll(timeout_ms=0)
|
|
1180
1176
|
if self.is_ready(node_id):
|
|
1181
1177
|
return True
|
|
1182
1178
|
|
|
1183
|
-
while not self.is_ready(node_id) and
|
|
1179
|
+
while not self.is_ready(node_id) and not timer.expired:
|
|
1184
1180
|
if self.connection_failed(node_id):
|
|
1185
1181
|
raise Errors.KafkaConnectionError("Connection to %s failed." % (node_id,))
|
|
1186
1182
|
self.maybe_connect(node_id)
|
|
1187
|
-
self.poll(timeout_ms=
|
|
1183
|
+
self.poll(timeout_ms=timer.timeout_ms)
|
|
1188
1184
|
return self.is_ready(node_id)
|
|
1189
1185
|
|
|
1190
1186
|
def send_and_receive(self, node_id, request):
|
|
@@ -3,13 +3,15 @@ from __future__ import absolute_import
|
|
|
3
3
|
import collections
|
|
4
4
|
import copy
|
|
5
5
|
import logging
|
|
6
|
+
import random
|
|
7
|
+
import re
|
|
6
8
|
import threading
|
|
7
9
|
import time
|
|
8
10
|
|
|
9
11
|
from kafka.vendor import six
|
|
10
12
|
|
|
11
13
|
from kafka import errors as Errors
|
|
12
|
-
from kafka.conn import
|
|
14
|
+
from kafka.conn import get_ip_port_afi
|
|
13
15
|
from kafka.future import Future
|
|
14
16
|
from kafka.structs import BrokerMetadata, PartitionMetadata, TopicPartition
|
|
15
17
|
|
|
@@ -422,3 +424,24 @@ class ClusterMetadata(object):
|
|
|
422
424
|
def __str__(self):
|
|
423
425
|
return 'ClusterMetadata(brokers: %d, topics: %d, coordinators: %d)' % \
|
|
424
426
|
(len(self._brokers), len(self._partitions), len(self._coordinators))
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def collect_hosts(hosts, randomize=True):
|
|
430
|
+
"""
|
|
431
|
+
Collects a comma-separated set of hosts (host:port) and optionally
|
|
432
|
+
randomize the returned list.
|
|
433
|
+
"""
|
|
434
|
+
|
|
435
|
+
if isinstance(hosts, six.string_types):
|
|
436
|
+
hosts = hosts.strip().split(',')
|
|
437
|
+
|
|
438
|
+
result = []
|
|
439
|
+
for host_port in hosts:
|
|
440
|
+
# ignore leading SECURITY_PROTOCOL:// to mimic java client
|
|
441
|
+
host_port = re.sub('^.*://', '', host_port)
|
|
442
|
+
host, port, afi = get_ip_port_afi(host_port)
|
|
443
|
+
result.append((host, port, afi))
|
|
444
|
+
|
|
445
|
+
if randomize:
|
|
446
|
+
random.shuffle(result)
|
|
447
|
+
return result
|
|
@@ -4,7 +4,7 @@ import copy
|
|
|
4
4
|
import errno
|
|
5
5
|
import io
|
|
6
6
|
import logging
|
|
7
|
-
from random import
|
|
7
|
+
from random import uniform
|
|
8
8
|
|
|
9
9
|
# selectors in stdlib as of py3.4
|
|
10
10
|
try:
|
|
@@ -1496,32 +1496,6 @@ def get_ip_port_afi(host_and_port_str):
|
|
|
1496
1496
|
return host, port, af
|
|
1497
1497
|
|
|
1498
1498
|
|
|
1499
|
-
def collect_hosts(hosts, randomize=True):
|
|
1500
|
-
"""
|
|
1501
|
-
Collects a comma-separated set of hosts (host:port) and optionally
|
|
1502
|
-
randomize the returned list.
|
|
1503
|
-
"""
|
|
1504
|
-
|
|
1505
|
-
if isinstance(hosts, six.string_types):
|
|
1506
|
-
hosts = hosts.strip().split(',')
|
|
1507
|
-
|
|
1508
|
-
result = []
|
|
1509
|
-
afi = socket.AF_INET
|
|
1510
|
-
for host_port in hosts:
|
|
1511
|
-
|
|
1512
|
-
host, port, afi = get_ip_port_afi(host_port)
|
|
1513
|
-
|
|
1514
|
-
if port < 0:
|
|
1515
|
-
port = DEFAULT_KAFKA_PORT
|
|
1516
|
-
|
|
1517
|
-
result.append((host, port, afi))
|
|
1518
|
-
|
|
1519
|
-
if randomize:
|
|
1520
|
-
shuffle(result)
|
|
1521
|
-
|
|
1522
|
-
return result
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
1499
|
def is_inet_4_or_6(gai):
|
|
1526
1500
|
"""Given a getaddrinfo struct, return True iff ipv4 or ipv6"""
|
|
1527
1501
|
return gai[0] in (socket.AF_INET, socket.AF_INET6)
|
|
@@ -19,7 +19,7 @@ from kafka.protocol.list_offsets import (
|
|
|
19
19
|
from kafka.record import MemoryRecords
|
|
20
20
|
from kafka.serializer import Deserializer
|
|
21
21
|
from kafka.structs import TopicPartition, OffsetAndMetadata, OffsetAndTimestamp
|
|
22
|
-
from kafka.util import
|
|
22
|
+
from kafka.util import Timer
|
|
23
23
|
|
|
24
24
|
log = logging.getLogger(__name__)
|
|
25
25
|
|
|
@@ -153,6 +153,7 @@ class Fetcher(six.Iterator):
|
|
|
153
153
|
future = self._client.send(node_id, request, wakeup=False)
|
|
154
154
|
future.add_callback(self._handle_fetch_response, node_id, fetch_offsets, time.time())
|
|
155
155
|
future.add_errback(self._handle_fetch_error, node_id)
|
|
156
|
+
future.add_both(self._clear_pending_fetch_request, node_id)
|
|
156
157
|
futures.append(future)
|
|
157
158
|
self._fetch_futures.extend(futures)
|
|
158
159
|
self._clean_done_fetch_futures()
|
|
@@ -229,7 +230,7 @@ class Fetcher(six.Iterator):
|
|
|
229
230
|
if not timestamps:
|
|
230
231
|
return {}
|
|
231
232
|
|
|
232
|
-
|
|
233
|
+
timer = Timer(timeout_ms, "Failed to get offsets by timestamps in %s ms" % (timeout_ms,))
|
|
233
234
|
timestamps = copy.copy(timestamps)
|
|
234
235
|
fetched_offsets = dict()
|
|
235
236
|
while True:
|
|
@@ -237,7 +238,7 @@ class Fetcher(six.Iterator):
|
|
|
237
238
|
return {}
|
|
238
239
|
|
|
239
240
|
future = self._send_list_offsets_requests(timestamps)
|
|
240
|
-
self._client.poll(future=future, timeout_ms=
|
|
241
|
+
self._client.poll(future=future, timeout_ms=timer.timeout_ms)
|
|
241
242
|
|
|
242
243
|
# Timeout w/o future completion
|
|
243
244
|
if not future.is_done:
|
|
@@ -255,12 +256,17 @@ class Fetcher(six.Iterator):
|
|
|
255
256
|
|
|
256
257
|
if future.exception.invalid_metadata or self._client.cluster.need_update:
|
|
257
258
|
refresh_future = self._client.cluster.request_update()
|
|
258
|
-
self._client.poll(future=refresh_future, timeout_ms=
|
|
259
|
+
self._client.poll(future=refresh_future, timeout_ms=timer.timeout_ms)
|
|
259
260
|
|
|
260
261
|
if not future.is_done:
|
|
261
262
|
break
|
|
262
263
|
else:
|
|
263
|
-
|
|
264
|
+
if timer.timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
|
|
265
|
+
time.sleep(self.config['retry_backoff_ms'] / 1000)
|
|
266
|
+
else:
|
|
267
|
+
time.sleep(timer.timeout_ms / 1000)
|
|
268
|
+
|
|
269
|
+
timer.maybe_raise()
|
|
264
270
|
|
|
265
271
|
raise Errors.KafkaTimeoutError(
|
|
266
272
|
"Failed to get offsets by timestamps in %s ms" % (timeout_ms,))
|
|
@@ -417,7 +423,7 @@ class Fetcher(six.Iterator):
|
|
|
417
423
|
expire_at = time.time() + self.config['request_timeout_ms'] / 1000
|
|
418
424
|
self._subscriptions.set_reset_pending(partitions, expire_at)
|
|
419
425
|
|
|
420
|
-
def on_success(result):
|
|
426
|
+
def on_success(timestamps_and_epochs, result):
|
|
421
427
|
fetched_offsets, partitions_to_retry = result
|
|
422
428
|
if partitions_to_retry:
|
|
423
429
|
self._subscriptions.reset_failed(partitions_to_retry, time.time() + self.config['retry_backoff_ms'] / 1000)
|
|
@@ -427,7 +433,7 @@ class Fetcher(six.Iterator):
|
|
|
427
433
|
ts, _epoch = timestamps_and_epochs[partition]
|
|
428
434
|
self._reset_offset_if_needed(partition, ts, offset.offset)
|
|
429
435
|
|
|
430
|
-
def on_failure(error):
|
|
436
|
+
def on_failure(partitions, error):
|
|
431
437
|
self._subscriptions.reset_failed(partitions, time.time() + self.config['retry_backoff_ms'] / 1000)
|
|
432
438
|
self._client.cluster.request_update()
|
|
433
439
|
|
|
@@ -438,8 +444,8 @@ class Fetcher(six.Iterator):
|
|
|
438
444
|
log.error("Discarding error in ListOffsetResponse because another error is pending: %s", error)
|
|
439
445
|
|
|
440
446
|
future = self._send_list_offsets_request(node_id, timestamps_and_epochs)
|
|
441
|
-
future.add_callback(on_success)
|
|
442
|
-
future.add_errback(on_failure)
|
|
447
|
+
future.add_callback(on_success, timestamps_and_epochs)
|
|
448
|
+
future.add_errback(on_failure, partitions)
|
|
443
449
|
|
|
444
450
|
def _send_list_offsets_requests(self, timestamps):
|
|
445
451
|
"""Fetch offsets for each partition in timestamps dict. This may send
|
|
@@ -643,36 +649,42 @@ class Fetcher(six.Iterator):
|
|
|
643
649
|
log.debug("Skipping fetch for partition %s because node %s is throttled",
|
|
644
650
|
partition, node_id)
|
|
645
651
|
|
|
652
|
+
elif not self._client.ready(node_id):
|
|
653
|
+
# Until we support send request queues, any attempt to send to a not-ready node will be
|
|
654
|
+
# immediately failed with NodeNotReadyError.
|
|
655
|
+
log.debug("Skipping fetch for partition %s because connection to leader node is not ready yet")
|
|
656
|
+
|
|
646
657
|
elif node_id in self._nodes_with_pending_fetch_requests:
|
|
647
658
|
log.debug("Skipping fetch for partition %s because there is a pending fetch request to node %s",
|
|
648
659
|
partition, node_id)
|
|
649
|
-
continue
|
|
650
660
|
|
|
651
|
-
if version < 5:
|
|
652
|
-
partition_info = (
|
|
653
|
-
partition.partition,
|
|
654
|
-
position.offset,
|
|
655
|
-
self.config['max_partition_fetch_bytes']
|
|
656
|
-
)
|
|
657
|
-
elif version <= 8:
|
|
658
|
-
partition_info = (
|
|
659
|
-
partition.partition,
|
|
660
|
-
position.offset,
|
|
661
|
-
-1, # log_start_offset is used internally by brokers / replicas only
|
|
662
|
-
self.config['max_partition_fetch_bytes'],
|
|
663
|
-
)
|
|
664
661
|
else:
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
662
|
+
# Leader is connected and does not have a pending fetch request
|
|
663
|
+
if version < 5:
|
|
664
|
+
partition_info = (
|
|
665
|
+
partition.partition,
|
|
666
|
+
position.offset,
|
|
667
|
+
self.config['max_partition_fetch_bytes']
|
|
668
|
+
)
|
|
669
|
+
elif version <= 8:
|
|
670
|
+
partition_info = (
|
|
671
|
+
partition.partition,
|
|
672
|
+
position.offset,
|
|
673
|
+
-1, # log_start_offset is used internally by brokers / replicas only
|
|
674
|
+
self.config['max_partition_fetch_bytes'],
|
|
675
|
+
)
|
|
676
|
+
else:
|
|
677
|
+
partition_info = (
|
|
678
|
+
partition.partition,
|
|
679
|
+
position.leader_epoch,
|
|
680
|
+
position.offset,
|
|
681
|
+
-1, # log_start_offset is used internally by brokers / replicas only
|
|
682
|
+
self.config['max_partition_fetch_bytes'],
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
fetchable[node_id][partition] = partition_info
|
|
686
|
+
log.debug("Adding fetch request for partition %s at offset %d",
|
|
687
|
+
partition, position.offset)
|
|
676
688
|
|
|
677
689
|
requests = {}
|
|
678
690
|
for node_id, next_partitions in six.iteritems(fetchable):
|
|
@@ -761,14 +773,18 @@ class Fetcher(six.Iterator):
|
|
|
761
773
|
|
|
762
774
|
if self._sensors:
|
|
763
775
|
self._sensors.fetch_latency.record((time.time() - send_time) * 1000)
|
|
764
|
-
self._nodes_with_pending_fetch_requests.remove(node_id)
|
|
765
776
|
|
|
766
777
|
def _handle_fetch_error(self, node_id, exception):
|
|
767
778
|
level = logging.INFO if isinstance(exception, Errors.Cancelled) else logging.ERROR
|
|
768
779
|
log.log(level, 'Fetch to node %s failed: %s', node_id, exception)
|
|
769
780
|
if node_id in self._session_handlers:
|
|
770
781
|
self._session_handlers[node_id].handle_error(exception)
|
|
771
|
-
|
|
782
|
+
|
|
783
|
+
def _clear_pending_fetch_request(self, node_id, _):
|
|
784
|
+
try:
|
|
785
|
+
self._nodes_with_pending_fetch_requests.remove(node_id)
|
|
786
|
+
except KeyError:
|
|
787
|
+
pass
|
|
772
788
|
|
|
773
789
|
def _parse_fetched_data(self, completed_fetch):
|
|
774
790
|
tp = completed_fetch.topic_partition
|
|
@@ -18,7 +18,7 @@ from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
|
|
|
18
18
|
from kafka.metrics import MetricConfig, Metrics
|
|
19
19
|
from kafka.protocol.list_offsets import OffsetResetStrategy
|
|
20
20
|
from kafka.structs import OffsetAndMetadata, TopicPartition
|
|
21
|
-
from kafka.util import
|
|
21
|
+
from kafka.util import Timer
|
|
22
22
|
from kafka.version import __version__
|
|
23
23
|
|
|
24
24
|
log = logging.getLogger(__name__)
|
|
@@ -679,41 +679,40 @@ class KafkaConsumer(six.Iterator):
|
|
|
679
679
|
assert not self._closed, 'KafkaConsumer is closed'
|
|
680
680
|
|
|
681
681
|
# Poll for new data until the timeout expires
|
|
682
|
-
|
|
682
|
+
timer = Timer(timeout_ms)
|
|
683
683
|
while not self._closed:
|
|
684
|
-
records = self._poll_once(
|
|
684
|
+
records = self._poll_once(timer, max_records, update_offsets=update_offsets)
|
|
685
685
|
if records:
|
|
686
686
|
return records
|
|
687
|
-
|
|
688
|
-
if inner_timeout_ms() <= 0:
|
|
687
|
+
elif timer.expired:
|
|
689
688
|
break
|
|
690
|
-
|
|
691
689
|
return {}
|
|
692
690
|
|
|
693
|
-
def _poll_once(self,
|
|
691
|
+
def _poll_once(self, timer, max_records, update_offsets=True):
|
|
694
692
|
"""Do one round of polling. In addition to checking for new data, this does
|
|
695
693
|
any needed heart-beating, auto-commits, and offset updates.
|
|
696
694
|
|
|
697
695
|
Arguments:
|
|
698
|
-
|
|
696
|
+
timer (Timer): The maximum time in milliseconds to block.
|
|
699
697
|
|
|
700
698
|
Returns:
|
|
701
699
|
dict: Map of topic to list of records (may be empty).
|
|
702
700
|
"""
|
|
703
|
-
|
|
704
|
-
if not self._coordinator.poll(timeout_ms=inner_timeout_ms()):
|
|
701
|
+
if not self._coordinator.poll(timeout_ms=timer.timeout_ms):
|
|
705
702
|
return {}
|
|
706
703
|
|
|
707
|
-
has_all_fetch_positions = self._update_fetch_positions(timeout_ms=
|
|
704
|
+
has_all_fetch_positions = self._update_fetch_positions(timeout_ms=timer.timeout_ms)
|
|
708
705
|
|
|
709
706
|
# If data is available already, e.g. from a previous network client
|
|
710
707
|
# poll() call to commit, then just return it immediately
|
|
711
708
|
records, partial = self._fetcher.fetched_records(max_records, update_offsets=update_offsets)
|
|
709
|
+
log.debug('Fetched records: %s, %s', records, partial)
|
|
712
710
|
# Before returning the fetched records, we can send off the
|
|
713
711
|
# next round of fetches and avoid block waiting for their
|
|
714
712
|
# responses to enable pipelining while the user is handling the
|
|
715
713
|
# fetched records.
|
|
716
714
|
if not partial:
|
|
715
|
+
log.debug("Sending fetches")
|
|
717
716
|
futures = self._fetcher.send_fetches()
|
|
718
717
|
if len(futures):
|
|
719
718
|
self._client.poll(timeout_ms=0)
|
|
@@ -723,7 +722,7 @@ class KafkaConsumer(six.Iterator):
|
|
|
723
722
|
|
|
724
723
|
# We do not want to be stuck blocking in poll if we are missing some positions
|
|
725
724
|
# since the offset lookup may be backing off after a failure
|
|
726
|
-
poll_timeout_ms =
|
|
725
|
+
poll_timeout_ms = min(timer.timeout_ms, self._coordinator.time_to_next_poll() * 1000)
|
|
727
726
|
if not has_all_fetch_positions:
|
|
728
727
|
poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
|
|
729
728
|
|
|
@@ -749,15 +748,14 @@ class KafkaConsumer(six.Iterator):
|
|
|
749
748
|
raise TypeError('partition must be a TopicPartition namedtuple')
|
|
750
749
|
assert self._subscription.is_assigned(partition), 'Partition is not assigned'
|
|
751
750
|
|
|
752
|
-
|
|
751
|
+
timer = Timer(timeout_ms)
|
|
753
752
|
position = self._subscription.assignment[partition].position
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
self._update_fetch_positions(timeout_ms=inner_timeout_ms())
|
|
753
|
+
while position is None:
|
|
754
|
+
# batch update fetch positions for any partitions without a valid position
|
|
755
|
+
if self._update_fetch_positions(timeout_ms=timer.timeout_ms):
|
|
758
756
|
position = self._subscription.assignment[partition].position
|
|
759
|
-
|
|
760
|
-
|
|
757
|
+
elif timer.expired:
|
|
758
|
+
return None
|
|
761
759
|
else:
|
|
762
760
|
return position.offset
|
|
763
761
|
|
|
@@ -16,7 +16,7 @@ from kafka.metrics import AnonMeasurable
|
|
|
16
16
|
from kafka.metrics.stats import Avg, Count, Max, Rate
|
|
17
17
|
from kafka.protocol.find_coordinator import FindCoordinatorRequest
|
|
18
18
|
from kafka.protocol.group import HeartbeatRequest, JoinGroupRequest, LeaveGroupRequest, SyncGroupRequest, DEFAULT_GENERATION_ID, UNKNOWN_MEMBER_ID
|
|
19
|
-
from kafka.util import
|
|
19
|
+
from kafka.util import Timer
|
|
20
20
|
|
|
21
21
|
log = logging.getLogger('kafka.coordinator')
|
|
22
22
|
|
|
@@ -256,9 +256,9 @@ class BaseCoordinator(object):
|
|
|
256
256
|
timeout_ms (numeric, optional): Maximum number of milliseconds to
|
|
257
257
|
block waiting to find coordinator. Default: None.
|
|
258
258
|
|
|
259
|
-
|
|
259
|
+
Returns: True is coordinator found before timeout_ms, else False
|
|
260
260
|
"""
|
|
261
|
-
|
|
261
|
+
timer = Timer(timeout_ms)
|
|
262
262
|
with self._client._lock, self._lock:
|
|
263
263
|
while self.coordinator_unknown():
|
|
264
264
|
|
|
@@ -272,27 +272,37 @@ class BaseCoordinator(object):
|
|
|
272
272
|
else:
|
|
273
273
|
self.coordinator_id = maybe_coordinator_id
|
|
274
274
|
self._client.maybe_connect(self.coordinator_id)
|
|
275
|
-
|
|
275
|
+
if timer.expired:
|
|
276
|
+
return False
|
|
277
|
+
else:
|
|
278
|
+
continue
|
|
276
279
|
else:
|
|
277
280
|
future = self.lookup_coordinator()
|
|
278
281
|
|
|
279
|
-
self._client.poll(future=future, timeout_ms=
|
|
282
|
+
self._client.poll(future=future, timeout_ms=timer.timeout_ms)
|
|
280
283
|
|
|
281
284
|
if not future.is_done:
|
|
282
|
-
|
|
285
|
+
return False
|
|
283
286
|
|
|
284
287
|
if future.failed():
|
|
285
288
|
if future.retriable():
|
|
286
289
|
if getattr(future.exception, 'invalid_metadata', False):
|
|
287
290
|
log.debug('Requesting metadata for group coordinator request: %s', future.exception)
|
|
288
291
|
metadata_update = self._client.cluster.request_update()
|
|
289
|
-
self._client.poll(future=metadata_update, timeout_ms=
|
|
292
|
+
self._client.poll(future=metadata_update, timeout_ms=timer.timeout_ms)
|
|
290
293
|
if not metadata_update.is_done:
|
|
291
|
-
|
|
294
|
+
return False
|
|
292
295
|
else:
|
|
293
|
-
|
|
296
|
+
if timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
|
|
297
|
+
time.sleep(self.config['retry_backoff_ms'] / 1000)
|
|
298
|
+
else:
|
|
299
|
+
time.sleep(timer.timeout_ms / 1000)
|
|
294
300
|
else:
|
|
295
301
|
raise future.exception # pylint: disable-msg=raising-bad-type
|
|
302
|
+
if timer.expired:
|
|
303
|
+
return False
|
|
304
|
+
else:
|
|
305
|
+
return True
|
|
296
306
|
|
|
297
307
|
def _reset_find_coordinator_future(self, result):
|
|
298
308
|
self._find_coordinator_future = None
|
|
@@ -407,21 +417,23 @@ class BaseCoordinator(object):
|
|
|
407
417
|
timeout_ms (numeric, optional): Maximum number of milliseconds to
|
|
408
418
|
block waiting to join group. Default: None.
|
|
409
419
|
|
|
410
|
-
|
|
420
|
+
Returns: True if group initialized before timeout_ms, else False
|
|
411
421
|
"""
|
|
412
422
|
if self.config['api_version'] < (0, 9):
|
|
413
423
|
raise Errors.UnsupportedVersionError('Group Coordinator APIs require 0.9+ broker')
|
|
414
|
-
|
|
415
|
-
self.ensure_coordinator_ready(timeout_ms=
|
|
424
|
+
timer = Timer(timeout_ms)
|
|
425
|
+
if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
|
|
426
|
+
return False
|
|
416
427
|
self._start_heartbeat_thread()
|
|
417
|
-
self.join_group(timeout_ms=
|
|
428
|
+
return self.join_group(timeout_ms=timer.timeout_ms)
|
|
418
429
|
|
|
419
430
|
def join_group(self, timeout_ms=None):
|
|
420
431
|
if self.config['api_version'] < (0, 9):
|
|
421
432
|
raise Errors.UnsupportedVersionError('Group Coordinator APIs require 0.9+ broker')
|
|
422
|
-
|
|
433
|
+
timer = Timer(timeout_ms)
|
|
423
434
|
while self.need_rejoin():
|
|
424
|
-
self.ensure_coordinator_ready(timeout_ms=
|
|
435
|
+
if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
|
|
436
|
+
return False
|
|
425
437
|
|
|
426
438
|
# call on_join_prepare if needed. We set a flag
|
|
427
439
|
# to make sure that we do not call it a second
|
|
@@ -434,7 +446,7 @@ class BaseCoordinator(object):
|
|
|
434
446
|
if not self.rejoining:
|
|
435
447
|
self._on_join_prepare(self._generation.generation_id,
|
|
436
448
|
self._generation.member_id,
|
|
437
|
-
timeout_ms=
|
|
449
|
+
timeout_ms=timer.timeout_ms)
|
|
438
450
|
self.rejoining = True
|
|
439
451
|
|
|
440
452
|
# fence off the heartbeat thread explicitly so that it cannot
|
|
@@ -449,16 +461,19 @@ class BaseCoordinator(object):
|
|
|
449
461
|
while not self.coordinator_unknown():
|
|
450
462
|
if not self._client.in_flight_request_count(self.coordinator_id):
|
|
451
463
|
break
|
|
452
|
-
|
|
464
|
+
poll_timeout_ms = 200 if timer.timeout_ms is None or timer.timeout_ms > 200 else timer.timeout_ms
|
|
465
|
+
self._client.poll(timeout_ms=poll_timeout_ms)
|
|
466
|
+
if timer.expired:
|
|
467
|
+
return False
|
|
453
468
|
else:
|
|
454
469
|
continue
|
|
455
470
|
|
|
456
471
|
future = self._initiate_join_group()
|
|
457
|
-
self._client.poll(future=future, timeout_ms=
|
|
472
|
+
self._client.poll(future=future, timeout_ms=timer.timeout_ms)
|
|
458
473
|
if future.is_done:
|
|
459
474
|
self._reset_join_group_future()
|
|
460
475
|
else:
|
|
461
|
-
|
|
476
|
+
return False
|
|
462
477
|
|
|
463
478
|
if future.succeeded():
|
|
464
479
|
self.rejoining = False
|
|
@@ -467,6 +482,7 @@ class BaseCoordinator(object):
|
|
|
467
482
|
self._generation.member_id,
|
|
468
483
|
self._generation.protocol,
|
|
469
484
|
future.value)
|
|
485
|
+
return True
|
|
470
486
|
else:
|
|
471
487
|
exception = future.exception
|
|
472
488
|
if isinstance(exception, (Errors.UnknownMemberIdError,
|
|
@@ -476,7 +492,13 @@ class BaseCoordinator(object):
|
|
|
476
492
|
continue
|
|
477
493
|
elif not future.retriable():
|
|
478
494
|
raise exception # pylint: disable-msg=raising-bad-type
|
|
479
|
-
|
|
495
|
+
elif timer.expired:
|
|
496
|
+
return False
|
|
497
|
+
else:
|
|
498
|
+
if timer.timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
|
|
499
|
+
time.sleep(self.config['retry_backoff_ms'] / 1000)
|
|
500
|
+
else:
|
|
501
|
+
time.sleep(timer.timeout_ms / 1000)
|
|
480
502
|
|
|
481
503
|
def _send_join_group_request(self):
|
|
482
504
|
"""Join the group and return the assignment for the next generation.
|