kafka-python 2.2.2__tar.gz → 2.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. {kafka_python-2.2.2 → kafka_python-2.2.4}/CHANGES.md +18 -0
  2. {kafka_python-2.2.2 → kafka_python-2.2.4}/PKG-INFO +1 -1
  3. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/client_async.py +7 -11
  4. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/cluster.py +24 -1
  5. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/conn.py +1 -27
  6. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/fetcher.py +52 -36
  7. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/group.py +17 -19
  8. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/base.py +42 -20
  9. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/consumer.py +72 -39
  10. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/errors.py +12 -12
  11. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/kafka.py +12 -24
  12. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/util.py +39 -19
  13. kafka_python-2.2.4/kafka/version.py +1 -0
  14. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/PKG-INFO +1 -1
  15. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_cluster.py +60 -1
  16. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_conn.py +1 -49
  17. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_fetcher.py +20 -10
  18. kafka_python-2.2.2/kafka/version.py +0 -1
  19. {kafka_python-2.2.2 → kafka_python-2.2.4}/AUTHORS.md +0 -0
  20. {kafka_python-2.2.2 → kafka_python-2.2.4}/LICENSE +0 -0
  21. {kafka_python-2.2.2 → kafka_python-2.2.4}/MANIFEST.in +0 -0
  22. {kafka_python-2.2.2 → kafka_python-2.2.4}/README.rst +0 -0
  23. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/__init__.py +0 -0
  24. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/__init__.py +0 -0
  25. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/acl_resource.py +0 -0
  26. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/client.py +0 -0
  27. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/config_resource.py +0 -0
  28. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/new_partitions.py +0 -0
  29. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/admin/new_topic.py +0 -0
  30. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/__init__.py +0 -0
  31. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/consumer_performance.py +0 -0
  32. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/load_example.py +0 -0
  33. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/producer_performance.py +0 -0
  34. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/record_batch_compose.py +0 -0
  35. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/record_batch_read.py +0 -0
  36. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/benchmarks/varint_speed.py +0 -0
  37. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/codec.py +0 -0
  38. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/__init__.py +0 -0
  39. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/consumer/subscription_state.py +0 -0
  40. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/__init__.py +0 -0
  41. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/__init__.py +0 -0
  42. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/abstract.py +0 -0
  43. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/range.py +0 -0
  44. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/roundrobin.py +0 -0
  45. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/__init__.py +0 -0
  46. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/partition_movements.py +0 -0
  47. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/sorted_set.py +0 -0
  48. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/assignors/sticky/sticky_assignor.py +0 -0
  49. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/heartbeat.py +0 -0
  50. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/coordinator/protocol.py +0 -0
  51. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/future.py +0 -0
  52. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/__init__.py +0 -0
  53. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/compound_stat.py +0 -0
  54. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/dict_reporter.py +0 -0
  55. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/kafka_metric.py +0 -0
  56. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/measurable.py +0 -0
  57. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/measurable_stat.py +0 -0
  58. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metric_config.py +0 -0
  59. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metric_name.py +0 -0
  60. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metrics.py +0 -0
  61. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/metrics_reporter.py +0 -0
  62. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/quota.py +0 -0
  63. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stat.py +0 -0
  64. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/__init__.py +0 -0
  65. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/avg.py +0 -0
  66. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/count.py +0 -0
  67. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/histogram.py +0 -0
  68. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/max_stat.py +0 -0
  69. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/min_stat.py +0 -0
  70. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/percentile.py +0 -0
  71. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/percentiles.py +0 -0
  72. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/rate.py +0 -0
  73. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/sampled_stat.py +0 -0
  74. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/sensor.py +0 -0
  75. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/metrics/stats/total.py +0 -0
  76. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/partitioner/__init__.py +0 -0
  77. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/partitioner/default.py +0 -0
  78. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/__init__.py +0 -0
  79. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/future.py +0 -0
  80. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/record_accumulator.py +0 -0
  81. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/sender.py +0 -0
  82. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/producer/transaction_manager.py +0 -0
  83. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/__init__.py +0 -0
  84. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/abstract.py +0 -0
  85. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/add_offsets_to_txn.py +0 -0
  86. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/add_partitions_to_txn.py +0 -0
  87. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/admin.py +0 -0
  88. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/api.py +0 -0
  89. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/api_versions.py +0 -0
  90. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/broker_api_versions.py +0 -0
  91. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/commit.py +0 -0
  92. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/end_txn.py +0 -0
  93. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/fetch.py +0 -0
  94. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/find_coordinator.py +0 -0
  95. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/frame.py +0 -0
  96. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/group.py +0 -0
  97. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/init_producer_id.py +0 -0
  98. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/list_offsets.py +0 -0
  99. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/message.py +0 -0
  100. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/metadata.py +0 -0
  101. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/offset_for_leader_epoch.py +0 -0
  102. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/parser.py +0 -0
  103. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/pickle.py +0 -0
  104. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/produce.py +0 -0
  105. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/sasl_authenticate.py +0 -0
  106. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/sasl_handshake.py +0 -0
  107. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/struct.py +0 -0
  108. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/txn_offset_commit.py +0 -0
  109. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/protocol/types.py +0 -0
  110. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/__init__.py +0 -0
  111. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/_crc32c.py +0 -0
  112. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/abc.py +0 -0
  113. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/default_records.py +0 -0
  114. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/legacy_records.py +0 -0
  115. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/memory_records.py +0 -0
  116. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/record/util.py +0 -0
  117. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/__init__.py +0 -0
  118. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/abc.py +0 -0
  119. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/gssapi.py +0 -0
  120. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/msk.py +0 -0
  121. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/oauth.py +0 -0
  122. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/plain.py +0 -0
  123. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/scram.py +0 -0
  124. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/sasl/sspi.py +0 -0
  125. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/serializer/__init__.py +0 -0
  126. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/serializer/abstract.py +0 -0
  127. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/socks5_wrapper.py +0 -0
  128. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/structs.py +0 -0
  129. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/__init__.py +0 -0
  130. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/enum34.py +0 -0
  131. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/selectors34.py +0 -0
  132. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/six.py +0 -0
  133. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka/vendor/socketpair.py +0 -0
  134. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/SOURCES.txt +0 -0
  135. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/dependency_links.txt +0 -0
  136. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/requires.txt +0 -0
  137. {kafka_python-2.2.2 → kafka_python-2.2.4}/kafka_python.egg-info/top_level.txt +0 -0
  138. {kafka_python-2.2.2 → kafka_python-2.2.4}/pyproject.toml +0 -0
  139. {kafka_python-2.2.2 → kafka_python-2.2.4}/setup.cfg +0 -0
  140. {kafka_python-2.2.2 → kafka_python-2.2.4}/setup.py +0 -0
  141. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/__init__.py +0 -0
  142. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/conftest.py +0 -0
  143. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/fixtures.py +0 -0
  144. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_admin_integration.py +0 -0
  145. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_consumer_group.py +0 -0
  146. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_consumer_integration.py +0 -0
  147. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_producer_integration.py +0 -0
  148. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/integration/test_sasl_integration.py +0 -0
  149. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_acl_comparisons.py +0 -0
  150. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_admin.py +0 -0
  151. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_api_object_implementation.py +0 -0
  152. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_assignors.py +0 -0
  153. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_client_async.py +0 -0
  154. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_codec.py +0 -0
  155. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_consumer.py +0 -0
  156. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_coordinator.py +0 -0
  157. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_metrics.py +0 -0
  158. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_object_conversion.py +0 -0
  159. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_package.py +0 -0
  160. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_partition_movements.py +0 -0
  161. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_partitioner.py +0 -0
  162. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_producer.py +0 -0
  163. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_protocol.py +0 -0
  164. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_record_accumulator.py +0 -0
  165. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_sender.py +0 -0
  166. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_subscription_state.py +0 -0
  167. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/test_util.py +0 -0
  168. {kafka_python-2.2.2 → kafka_python-2.2.4}/test/testutil.py +0 -0
@@ -1,3 +1,16 @@
1
+ # 2.2.4 (May 3, 2025)
2
+
3
+ Fixes
4
+ * Do not `reset_generation` after RebalanceInProgressError; improve CommitFailed error messages (#2614)
5
+ * Fix KafkaConsumer.poll() with zero timeout (#2613)
6
+ * Fix Fetch._reset_offsets_async() KeyError when fetching from multiple nodes (#2612)
7
+
8
+ # 2.2.3 (May 1, 2025)
9
+
10
+ Fixes
11
+ * Ignore leading SECURITY_PROTOCOL:// in bootstrap_servers (#2608)
12
+ * Only create fetch requests for ready nodes (#2607)
13
+
1
14
  # 2.2.2 (Apr 30, 2025)
2
15
 
3
16
  Fixes
@@ -58,6 +71,11 @@ Tests
58
71
  * py2 test fixups
59
72
  * Drop unused KafkaClient import from `test_fetcher`
60
73
 
74
+ # 2.1.6 (May 2, 2025)
75
+
76
+ Fixes
77
+ * Only create fetch requests for ready nodes (#2607)
78
+
61
79
  # 2.1.5 (Apr 4, 2025)
62
80
 
63
81
  Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kafka-python
3
- Version: 2.2.2
3
+ Version: 2.2.4
4
4
  Summary: Pure Python client for Apache Kafka
5
5
  Author-email: Dana Powers <dana.powers@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/dpkp/kafka-python
@@ -27,7 +27,7 @@ from kafka.metrics.stats import Avg, Count, Rate
27
27
  from kafka.metrics.stats.rate import TimeUnit
28
28
  from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS
29
29
  from kafka.protocol.metadata import MetadataRequest
30
- from kafka.util import Dict, WeakMethod, ensure_valid_topic_name, timeout_ms_fn
30
+ from kafka.util import Dict, Timer, WeakMethod, ensure_valid_topic_name
31
31
  # Although this looks unused, it actually monkey-patches socket.socketpair()
32
32
  # and should be left in as long as we're using socket.socketpair() in this file
33
33
  from kafka.vendor import socketpair # noqa: F401
@@ -645,12 +645,8 @@ class KafkaClient(object):
645
645
  """
646
646
  if not isinstance(timeout_ms, (int, float, type(None))):
647
647
  raise TypeError('Invalid type for timeout: %s' % type(timeout_ms))
648
+ timer = Timer(timeout_ms)
648
649
 
649
- begin = time.time()
650
- if timeout_ms is not None:
651
- timeout_at = begin + (timeout_ms / 1000)
652
- else:
653
- timeout_at = begin + (self.config['request_timeout_ms'] / 1000)
654
650
  # Loop for futures, break after first loop if None
655
651
  responses = []
656
652
  while True:
@@ -675,7 +671,7 @@ class KafkaClient(object):
675
671
  if future is not None and future.is_done:
676
672
  timeout = 0
677
673
  else:
678
- user_timeout_ms = 1000 * max(0, timeout_at - time.time())
674
+ user_timeout_ms = timer.timeout_ms if timeout_ms is not None else self.config['request_timeout_ms']
679
675
  idle_connection_timeout_ms = self._idle_expiry_manager.next_check_ms()
680
676
  request_timeout_ms = self._next_ifr_request_timeout_ms()
681
677
  log.debug("Timeouts: user %f, metadata %f, idle connection %f, request %f", user_timeout_ms, metadata_timeout_ms, idle_connection_timeout_ms, request_timeout_ms)
@@ -698,7 +694,7 @@ class KafkaClient(object):
698
694
  break
699
695
  elif future.is_done:
700
696
  break
701
- elif timeout_ms is not None and time.time() >= timeout_at:
697
+ elif timeout_ms is not None and timer.expired:
702
698
  break
703
699
 
704
700
  return responses
@@ -1175,16 +1171,16 @@ class KafkaClient(object):
1175
1171
  This method is useful for implementing blocking behaviour on top of the non-blocking `NetworkClient`, use it with
1176
1172
  care.
1177
1173
  """
1178
- inner_timeout_ms = timeout_ms_fn(timeout_ms, None)
1174
+ timer = Timer(timeout_ms)
1179
1175
  self.poll(timeout_ms=0)
1180
1176
  if self.is_ready(node_id):
1181
1177
  return True
1182
1178
 
1183
- while not self.is_ready(node_id) and inner_timeout_ms() > 0:
1179
+ while not self.is_ready(node_id) and not timer.expired:
1184
1180
  if self.connection_failed(node_id):
1185
1181
  raise Errors.KafkaConnectionError("Connection to %s failed." % (node_id,))
1186
1182
  self.maybe_connect(node_id)
1187
- self.poll(timeout_ms=inner_timeout_ms())
1183
+ self.poll(timeout_ms=timer.timeout_ms)
1188
1184
  return self.is_ready(node_id)
1189
1185
 
1190
1186
  def send_and_receive(self, node_id, request):
@@ -3,13 +3,15 @@ from __future__ import absolute_import
3
3
  import collections
4
4
  import copy
5
5
  import logging
6
+ import random
7
+ import re
6
8
  import threading
7
9
  import time
8
10
 
9
11
  from kafka.vendor import six
10
12
 
11
13
  from kafka import errors as Errors
12
- from kafka.conn import collect_hosts
14
+ from kafka.conn import get_ip_port_afi
13
15
  from kafka.future import Future
14
16
  from kafka.structs import BrokerMetadata, PartitionMetadata, TopicPartition
15
17
 
@@ -422,3 +424,24 @@ class ClusterMetadata(object):
422
424
  def __str__(self):
423
425
  return 'ClusterMetadata(brokers: %d, topics: %d, coordinators: %d)' % \
424
426
  (len(self._brokers), len(self._partitions), len(self._coordinators))
427
+
428
+
429
+ def collect_hosts(hosts, randomize=True):
430
+ """
431
+ Collects a comma-separated set of hosts (host:port) and optionally
432
+ randomize the returned list.
433
+ """
434
+
435
+ if isinstance(hosts, six.string_types):
436
+ hosts = hosts.strip().split(',')
437
+
438
+ result = []
439
+ for host_port in hosts:
440
+ # ignore leading SECURITY_PROTOCOL:// to mimic java client
441
+ host_port = re.sub('^.*://', '', host_port)
442
+ host, port, afi = get_ip_port_afi(host_port)
443
+ result.append((host, port, afi))
444
+
445
+ if randomize:
446
+ random.shuffle(result)
447
+ return result
@@ -4,7 +4,7 @@ import copy
4
4
  import errno
5
5
  import io
6
6
  import logging
7
- from random import shuffle, uniform
7
+ from random import uniform
8
8
 
9
9
  # selectors in stdlib as of py3.4
10
10
  try:
@@ -1496,32 +1496,6 @@ def get_ip_port_afi(host_and_port_str):
1496
1496
  return host, port, af
1497
1497
 
1498
1498
 
1499
- def collect_hosts(hosts, randomize=True):
1500
- """
1501
- Collects a comma-separated set of hosts (host:port) and optionally
1502
- randomize the returned list.
1503
- """
1504
-
1505
- if isinstance(hosts, six.string_types):
1506
- hosts = hosts.strip().split(',')
1507
-
1508
- result = []
1509
- afi = socket.AF_INET
1510
- for host_port in hosts:
1511
-
1512
- host, port, afi = get_ip_port_afi(host_port)
1513
-
1514
- if port < 0:
1515
- port = DEFAULT_KAFKA_PORT
1516
-
1517
- result.append((host, port, afi))
1518
-
1519
- if randomize:
1520
- shuffle(result)
1521
-
1522
- return result
1523
-
1524
-
1525
1499
  def is_inet_4_or_6(gai):
1526
1500
  """Given a getaddrinfo struct, return True iff ipv4 or ipv6"""
1527
1501
  return gai[0] in (socket.AF_INET, socket.AF_INET6)
@@ -19,7 +19,7 @@ from kafka.protocol.list_offsets import (
19
19
  from kafka.record import MemoryRecords
20
20
  from kafka.serializer import Deserializer
21
21
  from kafka.structs import TopicPartition, OffsetAndMetadata, OffsetAndTimestamp
22
- from kafka.util import timeout_ms_fn
22
+ from kafka.util import Timer
23
23
 
24
24
  log = logging.getLogger(__name__)
25
25
 
@@ -153,6 +153,7 @@ class Fetcher(six.Iterator):
153
153
  future = self._client.send(node_id, request, wakeup=False)
154
154
  future.add_callback(self._handle_fetch_response, node_id, fetch_offsets, time.time())
155
155
  future.add_errback(self._handle_fetch_error, node_id)
156
+ future.add_both(self._clear_pending_fetch_request, node_id)
156
157
  futures.append(future)
157
158
  self._fetch_futures.extend(futures)
158
159
  self._clean_done_fetch_futures()
@@ -229,7 +230,7 @@ class Fetcher(six.Iterator):
229
230
  if not timestamps:
230
231
  return {}
231
232
 
232
- inner_timeout_ms = timeout_ms_fn(timeout_ms, 'Timeout fetching offsets')
233
+ timer = Timer(timeout_ms, "Failed to get offsets by timestamps in %s ms" % (timeout_ms,))
233
234
  timestamps = copy.copy(timestamps)
234
235
  fetched_offsets = dict()
235
236
  while True:
@@ -237,7 +238,7 @@ class Fetcher(six.Iterator):
237
238
  return {}
238
239
 
239
240
  future = self._send_list_offsets_requests(timestamps)
240
- self._client.poll(future=future, timeout_ms=inner_timeout_ms())
241
+ self._client.poll(future=future, timeout_ms=timer.timeout_ms)
241
242
 
242
243
  # Timeout w/o future completion
243
244
  if not future.is_done:
@@ -255,12 +256,17 @@ class Fetcher(six.Iterator):
255
256
 
256
257
  if future.exception.invalid_metadata or self._client.cluster.need_update:
257
258
  refresh_future = self._client.cluster.request_update()
258
- self._client.poll(future=refresh_future, timeout_ms=inner_timeout_ms())
259
+ self._client.poll(future=refresh_future, timeout_ms=timer.timeout_ms)
259
260
 
260
261
  if not future.is_done:
261
262
  break
262
263
  else:
263
- time.sleep(inner_timeout_ms(self.config['retry_backoff_ms']) / 1000)
264
+ if timer.timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
265
+ time.sleep(self.config['retry_backoff_ms'] / 1000)
266
+ else:
267
+ time.sleep(timer.timeout_ms / 1000)
268
+
269
+ timer.maybe_raise()
264
270
 
265
271
  raise Errors.KafkaTimeoutError(
266
272
  "Failed to get offsets by timestamps in %s ms" % (timeout_ms,))
@@ -417,7 +423,7 @@ class Fetcher(six.Iterator):
417
423
  expire_at = time.time() + self.config['request_timeout_ms'] / 1000
418
424
  self._subscriptions.set_reset_pending(partitions, expire_at)
419
425
 
420
- def on_success(result):
426
+ def on_success(timestamps_and_epochs, result):
421
427
  fetched_offsets, partitions_to_retry = result
422
428
  if partitions_to_retry:
423
429
  self._subscriptions.reset_failed(partitions_to_retry, time.time() + self.config['retry_backoff_ms'] / 1000)
@@ -427,7 +433,7 @@ class Fetcher(six.Iterator):
427
433
  ts, _epoch = timestamps_and_epochs[partition]
428
434
  self._reset_offset_if_needed(partition, ts, offset.offset)
429
435
 
430
- def on_failure(error):
436
+ def on_failure(partitions, error):
431
437
  self._subscriptions.reset_failed(partitions, time.time() + self.config['retry_backoff_ms'] / 1000)
432
438
  self._client.cluster.request_update()
433
439
 
@@ -438,8 +444,8 @@ class Fetcher(six.Iterator):
438
444
  log.error("Discarding error in ListOffsetResponse because another error is pending: %s", error)
439
445
 
440
446
  future = self._send_list_offsets_request(node_id, timestamps_and_epochs)
441
- future.add_callback(on_success)
442
- future.add_errback(on_failure)
447
+ future.add_callback(on_success, timestamps_and_epochs)
448
+ future.add_errback(on_failure, partitions)
443
449
 
444
450
  def _send_list_offsets_requests(self, timestamps):
445
451
  """Fetch offsets for each partition in timestamps dict. This may send
@@ -643,36 +649,42 @@ class Fetcher(six.Iterator):
643
649
  log.debug("Skipping fetch for partition %s because node %s is throttled",
644
650
  partition, node_id)
645
651
 
652
+ elif not self._client.ready(node_id):
653
+ # Until we support send request queues, any attempt to send to a not-ready node will be
654
+ # immediately failed with NodeNotReadyError.
655
+ log.debug("Skipping fetch for partition %s because connection to leader node is not ready yet")
656
+
646
657
  elif node_id in self._nodes_with_pending_fetch_requests:
647
658
  log.debug("Skipping fetch for partition %s because there is a pending fetch request to node %s",
648
659
  partition, node_id)
649
- continue
650
660
 
651
- if version < 5:
652
- partition_info = (
653
- partition.partition,
654
- position.offset,
655
- self.config['max_partition_fetch_bytes']
656
- )
657
- elif version <= 8:
658
- partition_info = (
659
- partition.partition,
660
- position.offset,
661
- -1, # log_start_offset is used internally by brokers / replicas only
662
- self.config['max_partition_fetch_bytes'],
663
- )
664
661
  else:
665
- partition_info = (
666
- partition.partition,
667
- position.leader_epoch,
668
- position.offset,
669
- -1, # log_start_offset is used internally by brokers / replicas only
670
- self.config['max_partition_fetch_bytes'],
671
- )
672
-
673
- fetchable[node_id][partition] = partition_info
674
- log.debug("Adding fetch request for partition %s at offset %d",
675
- partition, position.offset)
662
+ # Leader is connected and does not have a pending fetch request
663
+ if version < 5:
664
+ partition_info = (
665
+ partition.partition,
666
+ position.offset,
667
+ self.config['max_partition_fetch_bytes']
668
+ )
669
+ elif version <= 8:
670
+ partition_info = (
671
+ partition.partition,
672
+ position.offset,
673
+ -1, # log_start_offset is used internally by brokers / replicas only
674
+ self.config['max_partition_fetch_bytes'],
675
+ )
676
+ else:
677
+ partition_info = (
678
+ partition.partition,
679
+ position.leader_epoch,
680
+ position.offset,
681
+ -1, # log_start_offset is used internally by brokers / replicas only
682
+ self.config['max_partition_fetch_bytes'],
683
+ )
684
+
685
+ fetchable[node_id][partition] = partition_info
686
+ log.debug("Adding fetch request for partition %s at offset %d",
687
+ partition, position.offset)
676
688
 
677
689
  requests = {}
678
690
  for node_id, next_partitions in six.iteritems(fetchable):
@@ -761,14 +773,18 @@ class Fetcher(six.Iterator):
761
773
 
762
774
  if self._sensors:
763
775
  self._sensors.fetch_latency.record((time.time() - send_time) * 1000)
764
- self._nodes_with_pending_fetch_requests.remove(node_id)
765
776
 
766
777
  def _handle_fetch_error(self, node_id, exception):
767
778
  level = logging.INFO if isinstance(exception, Errors.Cancelled) else logging.ERROR
768
779
  log.log(level, 'Fetch to node %s failed: %s', node_id, exception)
769
780
  if node_id in self._session_handlers:
770
781
  self._session_handlers[node_id].handle_error(exception)
771
- self._nodes_with_pending_fetch_requests.remove(node_id)
782
+
783
+ def _clear_pending_fetch_request(self, node_id, _):
784
+ try:
785
+ self._nodes_with_pending_fetch_requests.remove(node_id)
786
+ except KeyError:
787
+ pass
772
788
 
773
789
  def _parse_fetched_data(self, completed_fetch):
774
790
  tp = completed_fetch.topic_partition
@@ -18,7 +18,7 @@ from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
18
18
  from kafka.metrics import MetricConfig, Metrics
19
19
  from kafka.protocol.list_offsets import OffsetResetStrategy
20
20
  from kafka.structs import OffsetAndMetadata, TopicPartition
21
- from kafka.util import timeout_ms_fn
21
+ from kafka.util import Timer
22
22
  from kafka.version import __version__
23
23
 
24
24
  log = logging.getLogger(__name__)
@@ -679,41 +679,40 @@ class KafkaConsumer(six.Iterator):
679
679
  assert not self._closed, 'KafkaConsumer is closed'
680
680
 
681
681
  # Poll for new data until the timeout expires
682
- inner_timeout_ms = timeout_ms_fn(timeout_ms, None)
682
+ timer = Timer(timeout_ms)
683
683
  while not self._closed:
684
- records = self._poll_once(inner_timeout_ms(), max_records, update_offsets=update_offsets)
684
+ records = self._poll_once(timer, max_records, update_offsets=update_offsets)
685
685
  if records:
686
686
  return records
687
-
688
- if inner_timeout_ms() <= 0:
687
+ elif timer.expired:
689
688
  break
690
-
691
689
  return {}
692
690
 
693
- def _poll_once(self, timeout_ms, max_records, update_offsets=True):
691
+ def _poll_once(self, timer, max_records, update_offsets=True):
694
692
  """Do one round of polling. In addition to checking for new data, this does
695
693
  any needed heart-beating, auto-commits, and offset updates.
696
694
 
697
695
  Arguments:
698
- timeout_ms (int): The maximum time in milliseconds to block.
696
+ timer (Timer): The maximum time in milliseconds to block.
699
697
 
700
698
  Returns:
701
699
  dict: Map of topic to list of records (may be empty).
702
700
  """
703
- inner_timeout_ms = timeout_ms_fn(timeout_ms, None)
704
- if not self._coordinator.poll(timeout_ms=inner_timeout_ms()):
701
+ if not self._coordinator.poll(timeout_ms=timer.timeout_ms):
705
702
  return {}
706
703
 
707
- has_all_fetch_positions = self._update_fetch_positions(timeout_ms=inner_timeout_ms())
704
+ has_all_fetch_positions = self._update_fetch_positions(timeout_ms=timer.timeout_ms)
708
705
 
709
706
  # If data is available already, e.g. from a previous network client
710
707
  # poll() call to commit, then just return it immediately
711
708
  records, partial = self._fetcher.fetched_records(max_records, update_offsets=update_offsets)
709
+ log.debug('Fetched records: %s, %s', records, partial)
712
710
  # Before returning the fetched records, we can send off the
713
711
  # next round of fetches and avoid block waiting for their
714
712
  # responses to enable pipelining while the user is handling the
715
713
  # fetched records.
716
714
  if not partial:
715
+ log.debug("Sending fetches")
717
716
  futures = self._fetcher.send_fetches()
718
717
  if len(futures):
719
718
  self._client.poll(timeout_ms=0)
@@ -723,7 +722,7 @@ class KafkaConsumer(six.Iterator):
723
722
 
724
723
  # We do not want to be stuck blocking in poll if we are missing some positions
725
724
  # since the offset lookup may be backing off after a failure
726
- poll_timeout_ms = inner_timeout_ms(self._coordinator.time_to_next_poll() * 1000)
725
+ poll_timeout_ms = min(timer.timeout_ms, self._coordinator.time_to_next_poll() * 1000)
727
726
  if not has_all_fetch_positions:
728
727
  poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
729
728
 
@@ -749,15 +748,14 @@ class KafkaConsumer(six.Iterator):
749
748
  raise TypeError('partition must be a TopicPartition namedtuple')
750
749
  assert self._subscription.is_assigned(partition), 'Partition is not assigned'
751
750
 
752
- inner_timeout_ms = timeout_ms_fn(timeout_ms, 'Timeout retrieving partition position')
751
+ timer = Timer(timeout_ms)
753
752
  position = self._subscription.assignment[partition].position
754
- try:
755
- while position is None:
756
- # batch update fetch positions for any partitions without a valid position
757
- self._update_fetch_positions(timeout_ms=inner_timeout_ms())
753
+ while position is None:
754
+ # batch update fetch positions for any partitions without a valid position
755
+ if self._update_fetch_positions(timeout_ms=timer.timeout_ms):
758
756
  position = self._subscription.assignment[partition].position
759
- except KafkaTimeoutError:
760
- return None
757
+ elif timer.expired:
758
+ return None
761
759
  else:
762
760
  return position.offset
763
761
 
@@ -16,7 +16,7 @@ from kafka.metrics import AnonMeasurable
16
16
  from kafka.metrics.stats import Avg, Count, Max, Rate
17
17
  from kafka.protocol.find_coordinator import FindCoordinatorRequest
18
18
  from kafka.protocol.group import HeartbeatRequest, JoinGroupRequest, LeaveGroupRequest, SyncGroupRequest, DEFAULT_GENERATION_ID, UNKNOWN_MEMBER_ID
19
- from kafka.util import timeout_ms_fn
19
+ from kafka.util import Timer
20
20
 
21
21
  log = logging.getLogger('kafka.coordinator')
22
22
 
@@ -256,9 +256,9 @@ class BaseCoordinator(object):
256
256
  timeout_ms (numeric, optional): Maximum number of milliseconds to
257
257
  block waiting to find coordinator. Default: None.
258
258
 
259
- Raises: KafkaTimeoutError if timeout_ms is not None
259
+ Returns: True is coordinator found before timeout_ms, else False
260
260
  """
261
- inner_timeout_ms = timeout_ms_fn(timeout_ms, 'Timeout attempting to find group coordinator')
261
+ timer = Timer(timeout_ms)
262
262
  with self._client._lock, self._lock:
263
263
  while self.coordinator_unknown():
264
264
 
@@ -272,27 +272,37 @@ class BaseCoordinator(object):
272
272
  else:
273
273
  self.coordinator_id = maybe_coordinator_id
274
274
  self._client.maybe_connect(self.coordinator_id)
275
- continue
275
+ if timer.expired:
276
+ return False
277
+ else:
278
+ continue
276
279
  else:
277
280
  future = self.lookup_coordinator()
278
281
 
279
- self._client.poll(future=future, timeout_ms=inner_timeout_ms())
282
+ self._client.poll(future=future, timeout_ms=timer.timeout_ms)
280
283
 
281
284
  if not future.is_done:
282
- raise Errors.KafkaTimeoutError()
285
+ return False
283
286
 
284
287
  if future.failed():
285
288
  if future.retriable():
286
289
  if getattr(future.exception, 'invalid_metadata', False):
287
290
  log.debug('Requesting metadata for group coordinator request: %s', future.exception)
288
291
  metadata_update = self._client.cluster.request_update()
289
- self._client.poll(future=metadata_update, timeout_ms=inner_timeout_ms())
292
+ self._client.poll(future=metadata_update, timeout_ms=timer.timeout_ms)
290
293
  if not metadata_update.is_done:
291
- raise Errors.KafkaTimeoutError()
294
+ return False
292
295
  else:
293
- time.sleep(inner_timeout_ms(self.config['retry_backoff_ms']) / 1000)
296
+ if timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
297
+ time.sleep(self.config['retry_backoff_ms'] / 1000)
298
+ else:
299
+ time.sleep(timer.timeout_ms / 1000)
294
300
  else:
295
301
  raise future.exception # pylint: disable-msg=raising-bad-type
302
+ if timer.expired:
303
+ return False
304
+ else:
305
+ return True
296
306
 
297
307
  def _reset_find_coordinator_future(self, result):
298
308
  self._find_coordinator_future = None
@@ -407,21 +417,23 @@ class BaseCoordinator(object):
407
417
  timeout_ms (numeric, optional): Maximum number of milliseconds to
408
418
  block waiting to join group. Default: None.
409
419
 
410
- Raises: KafkaTimeoutError if timeout_ms is not None
420
+ Returns: True if group initialized before timeout_ms, else False
411
421
  """
412
422
  if self.config['api_version'] < (0, 9):
413
423
  raise Errors.UnsupportedVersionError('Group Coordinator APIs require 0.9+ broker')
414
- inner_timeout_ms = timeout_ms_fn(timeout_ms, 'Timeout attempting to join consumer group')
415
- self.ensure_coordinator_ready(timeout_ms=inner_timeout_ms())
424
+ timer = Timer(timeout_ms)
425
+ if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
426
+ return False
416
427
  self._start_heartbeat_thread()
417
- self.join_group(timeout_ms=inner_timeout_ms())
428
+ return self.join_group(timeout_ms=timer.timeout_ms)
418
429
 
419
430
  def join_group(self, timeout_ms=None):
420
431
  if self.config['api_version'] < (0, 9):
421
432
  raise Errors.UnsupportedVersionError('Group Coordinator APIs require 0.9+ broker')
422
- inner_timeout_ms = timeout_ms_fn(timeout_ms, 'Timeout attempting to join consumer group')
433
+ timer = Timer(timeout_ms)
423
434
  while self.need_rejoin():
424
- self.ensure_coordinator_ready(timeout_ms=inner_timeout_ms())
435
+ if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
436
+ return False
425
437
 
426
438
  # call on_join_prepare if needed. We set a flag
427
439
  # to make sure that we do not call it a second
@@ -434,7 +446,7 @@ class BaseCoordinator(object):
434
446
  if not self.rejoining:
435
447
  self._on_join_prepare(self._generation.generation_id,
436
448
  self._generation.member_id,
437
- timeout_ms=inner_timeout_ms())
449
+ timeout_ms=timer.timeout_ms)
438
450
  self.rejoining = True
439
451
 
440
452
  # fence off the heartbeat thread explicitly so that it cannot
@@ -449,16 +461,19 @@ class BaseCoordinator(object):
449
461
  while not self.coordinator_unknown():
450
462
  if not self._client.in_flight_request_count(self.coordinator_id):
451
463
  break
452
- self._client.poll(timeout_ms=inner_timeout_ms(200))
464
+ poll_timeout_ms = 200 if timer.timeout_ms is None or timer.timeout_ms > 200 else timer.timeout_ms
465
+ self._client.poll(timeout_ms=poll_timeout_ms)
466
+ if timer.expired:
467
+ return False
453
468
  else:
454
469
  continue
455
470
 
456
471
  future = self._initiate_join_group()
457
- self._client.poll(future=future, timeout_ms=inner_timeout_ms())
472
+ self._client.poll(future=future, timeout_ms=timer.timeout_ms)
458
473
  if future.is_done:
459
474
  self._reset_join_group_future()
460
475
  else:
461
- raise Errors.KafkaTimeoutError()
476
+ return False
462
477
 
463
478
  if future.succeeded():
464
479
  self.rejoining = False
@@ -467,6 +482,7 @@ class BaseCoordinator(object):
467
482
  self._generation.member_id,
468
483
  self._generation.protocol,
469
484
  future.value)
485
+ return True
470
486
  else:
471
487
  exception = future.exception
472
488
  if isinstance(exception, (Errors.UnknownMemberIdError,
@@ -476,7 +492,13 @@ class BaseCoordinator(object):
476
492
  continue
477
493
  elif not future.retriable():
478
494
  raise exception # pylint: disable-msg=raising-bad-type
479
- time.sleep(inner_timeout_ms(self.config['retry_backoff_ms']) / 1000)
495
+ elif timer.expired:
496
+ return False
497
+ else:
498
+ if timer.timeout_ms is None or timer.timeout_ms > self.config['retry_backoff_ms']:
499
+ time.sleep(self.config['retry_backoff_ms'] / 1000)
500
+ else:
501
+ time.sleep(timer.timeout_ms / 1000)
480
502
 
481
503
  def _send_join_group_request(self):
482
504
  """Join the group and return the assignment for the next generation.