kafka-python 2.2.16__py2.py3-none-any.whl → 2.2.17__py2.py3-none-any.whl

kafka/conn.py

@@ -326,6 +326,9 @@ class BrokerConnection(object):
         return True
 
     def _next_afi_sockaddr(self):
+        if self.config["socks5_proxy"] and Socks5Wrapper.use_remote_lookup(self.config["socks5_proxy"]):
+            return (socket.AF_UNSPEC, (self.host, self.port))
+
         if not self._gai:
             if not self._dns_lookup():
                 return
@@ -379,6 +382,7 @@ class BrokerConnection(object):
                 self._sock_afi, self._sock_addr = next_lookup
                 try:
                     if self.config["socks5_proxy"] is not None:
+                        log.debug('%s: initializing Socks5 proxy at %s', self, self.config["socks5_proxy"])
                         self._socks5_proxy = Socks5Wrapper(self.config["socks5_proxy"], self.afi)
                         self._sock = self._socks5_proxy.socket(self._sock_afi, socket.SOCK_STREAM)
                     else:
@@ -864,6 +868,8 @@ class BrokerConnection(object):
         if self.disconnected() or self.connecting():
             if len(self._gai) > 0:
                 return 0
+            elif self.config["socks5_proxy"] and Socks5Wrapper.use_remote_lookup(self.config["socks5_proxy"]):
+                return 0
             else:
                 time_waited = time.time() - self.last_attempt
                 return max(self._reconnect_backoff - time_waited, 0) * 1000
@@ -964,6 +970,7 @@ class BrokerConnection(object):
             # the socket fd from selectors cleanly.
             sock = self._sock
             self._sock = None
+            self._socks5_proxy = None
 
         # drop lock before state change callback and processing futures
         self.config['state_change_callback'](self.node_id, sock, self)

kafka/consumer/group.py

@@ -723,7 +723,9 @@ class KafkaConsumer(six.Iterator):
 
         # We do not want to be stuck blocking in poll if we are missing some positions
         # since the offset lookup may be backing off after a failure
-        poll_timeout_ms = min(timer.timeout_ms, self._coordinator.time_to_next_poll() * 1000)
+        poll_timeout_ms = timer.timeout_ms
+        if self.config['group_id'] is not None:
+            poll_timeout_ms = min(poll_timeout_ms, self._coordinator.time_to_next_poll() * 1000)
         if not has_all_fetch_positions:
             log.debug('poll: do not have all fetch positions...')
             poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
@@ -753,13 +755,12 @@ class KafkaConsumer(six.Iterator):
 
         timer = Timer(timeout_ms)
         position = self._subscription.assignment[partition].position
-        while position is None:
+        while position is None and not timer.expired:
             # batch update fetch positions for any partitions without a valid position
-            if self._update_fetch_positions(timeout_ms=timer.timeout_ms):
-                position = self._subscription.assignment[partition].position
-            if timer.expired:
-                return None
-        else:
+            self._update_fetch_positions(timeout_ms=timer.timeout_ms)
+            self._client.poll(timeout_ms=timer.timeout_ms)
+            position = self._subscription.assignment[partition].position
+        if position is not None:
             return position.offset
 
     def highwater(self, partition):

kafka/coordinator/base.py

@@ -1120,6 +1120,7 @@ class HeartbeatThread(threading.Thread):
                     self.coordinator._lock.wait(self.coordinator.config['retry_backoff_ms'] / 1000)
 
             elif not self.coordinator.connected():
+                self.coordinator._client.maybe_connect(self.coordinator.coordinator_id)
                 self.coordinator._client._lock.release()
                 self.coordinator._lock.wait(self.coordinator.config['retry_backoff_ms'] / 1000)
 

kafka/producer/kafka.py

@@ -134,10 +134,14 @@ class KafkaProducer(object):
         value_serializer (callable): used to convert user-supplied message
             values to bytes. If not None, called as f(value), should return
             bytes. Default: None.
+        transactional_id (str): Enable transactional producer with a unique
+            identifier. This will be used to identify the same producer
+            instance across process restarts. Default: None.
         enable_idempotence (bool): When set to True, the producer will ensure
             that exactly one copy of each message is written in the stream.
             If False, producer retries due to broker failures, etc., may write
-            duplicates of the retried message in the stream. Default: False.
+            duplicates of the retried message in the stream.
+            Default: True if `transactional_id` is provided, otherwise False.
 
             Note that enabling idempotence requires
             `max_in_flight_requests_per_connection` to be set to 1 and `retries`

kafka/socks5_wrapper.py

@@ -64,6 +64,15 @@ class Socks5Wrapper:
             log.warning("DNS lookup failed for proxy %s:%d, %r", host, port, ex)
             return []
 
+    @classmethod
+    def use_remote_lookup(cls, proxy_url):
+        if proxy_url is None:
+            return False
+        return urlparse(proxy_url).scheme == 'socks5h'
+
+    def _use_remote_lookup(self):
+        return self._proxy_url.scheme == 'socks5h'
+
     def socket(self, family, sock_type):
         """Open and record a socket.
 
@@ -187,7 +196,10 @@ class Socks5Wrapper:
                 return errno.ECONNREFUSED
 
         if self._state == ProxyConnectionStates.REQUEST_SUBMIT:
-            if self._target_afi == socket.AF_INET:
+            if self._use_remote_lookup():
+                addr_type = 3
+                addr_len = len(addr[0])
+            elif self._target_afi == socket.AF_INET:
                 addr_type = 1
                 addr_len = 4
             elif self._target_afi == socket.AF_INET6:
@@ -200,14 +212,28 @@ class Socks5Wrapper:
                 return errno.ECONNREFUSED
 
             self._buffer_out = struct.pack(
-                "!bbbb{}sh".format(addr_len),
+                "!bbbb",
                 5,  # version
                 1,  # command: connect
                 0,  # reserved
-                addr_type,  # 1 for ipv4, 4 for ipv6 address
-                socket.inet_pton(self._target_afi, addr[0]),  # either 4 or 16 bytes of actual address
-                addr[1],  # port
+                addr_type,  # 1 for ipv4, 4 for ipv6 address, 3 for domain name
             )
+            # Addr format depends on type
+            if addr_type == 3:
+                # len + domain name (no null terminator)
+                self._buffer_out += struct.pack(
+                    "!b{}s".format(addr_len),
+                    addr_len,
+                    addr[0].encode('ascii'),
+                )
+            else:
+                # either 4 (type 1) or 16 (type 4) bytes of actual address
+                self._buffer_out += struct.pack(
+                    "!{}s".format(addr_len),
+                    socket.inet_pton(self._target_afi, addr[0]),
+                )
+            self._buffer_out += struct.pack("!H", addr[1])  # port
+
             self._state = ProxyConnectionStates.REQUESTING
 
         if self._state == ProxyConnectionStates.REQUESTING:

kafka/version.py

@@ -1 +1 @@
-__version__ = '2.2.16'
+__version__ = '2.2.17'

{kafka_python-2.2.16.dist-info → kafka_python-2.2.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.16
+Version: 2.2.17
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.16.dist-info → kafka_python-2.2.17.dist-info}/RECORD

@@ -2,13 +2,13 @@ kafka/__init__.py,sha256=4dvHKZAxmD_4tfJ5wGcRV2X78vPcm8vsUoqceULevjA,1077
 kafka/client_async.py,sha256=R8q_rRpG3RrYrRmcZo7XgO2oSdpLJATNcq8w-1vIJ_8,56878
 kafka/cluster.py,sha256=B4tOZYhZaYrcGsyAtdA8yejFm9ue7ElJxn_pd6Xhdfk,16775
 kafka/codec.py,sha256=8NZpnehzNrhSBIjzbPVSvyFbSeLAqEntE7BfVHu-_9I,10036
-kafka/conn.py,sha256=La-xtpbM7k5ocswFJf3gm4fHNWzQV3fCG0YfeKFRYGI,69956
+kafka/conn.py,sha256=hDwKQ93zpZAWfACdiInEfcBSD6MpYCxVNoAjFneQW9Q,70406
 kafka/errors.py,sha256=qX2Fp0qawU_HBNcZCwB7EDCmx3C2PehrETi6qSEJHmk,33290
 kafka/future.py,sha256=ZQStbfUYIPJRrgMfAWxxjrIRVxsw4WCtSR0J0bkyGno,2847
-kafka/socks5_wrapper.py,sha256=6woOaCTJXJ5e89_zdyW5BjOpyE4rCbYFH-kd-FeuPuk,9827
+kafka/socks5_wrapper.py,sha256=h0Gag3xAOp8io2MfzeYLOiNvLRmq3rkmpZ6Aj-9uKTw,10716
 kafka/structs.py,sha256=SJGzmLdV21jZyQ7247k0WFy16UiusgTHK3I-e4qzI-E,3058
 kafka/util.py,sha256=WGqI5yT1yWGgHqSuRF9Fi8ejpiB53SurMy7ABkYxJ2g,4584
-kafka/version.py,sha256=UTmClSKnv3Rn1Qies89Pt3D8Dmn-myrPMLtPf-wbASY,23
+kafka/version.py,sha256=cwT2pG8_0ZVnZ1VhpzKqNri9AmJ_-isNazUrTQ3CI14,23
 kafka/admin/__init__.py,sha256=S_XxqyyV480_yXhttK79XZqNAmZyXRjspd3SoqYykE8,720
 kafka/admin/acl_resource.py,sha256=ak_dUsSni4SyP0ORbSKenZpwTy0Ykxq3FSt_9XgLR8k,8265
 kafka/admin/client.py,sha256=94UpHTsgzvhOoB6_1QLeKxvZKlStKfI96xuWyaY9_Sc,78814
@@ -24,10 +24,10 @@ kafka/benchmarks/record_batch_read.py,sha256=vlFaWU2YWI379n_2M8qieb_S2uHUWKV0Nqu
 kafka/benchmarks/varint_speed.py,sha256=s4CuvKgDZL-_zna5E3vM8RgHjhXuW6pcaO1z1WYZ_0Y,12585
 kafka/consumer/__init__.py,sha256=NDdvtyuJgFyQZahqL9i5sYXGP6rOMIXWwHQEaZ1fCcs,122
 kafka/consumer/fetcher.py,sha256=RlQLut54c5nOMl21neTJA2tmdsxIIPIX2Idu5Q-dYKY,69184
-kafka/consumer/group.py,sha256=kGvgqqdV9zrgCXr7jSbTs9sSJACjuAKehuxt0QYRHHU,58943
+kafka/consumer/group.py,sha256=rkjqEjWY72is-tRfm2mjAF1m6dACUP83PO_XGbkJu8I,59071
 kafka/consumer/subscription_state.py,sha256=bK-YTVbOzhy8OB206QAfXVuo7zPA9YqYXnrRRST369c,24289
 kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kafka/coordinator/base.py,sha256=hXfwtDkrHXHiNqjshCOa19js_2Y6ibLsdzDvJKGmcKc,54419
+kafka/coordinator/base.py,sha256=4GdPwC28S5a6e7B20fvzMVcL0YKemS43B4JiogeEd0E,54507
 kafka/coordinator/consumer.py,sha256=le4bGbHfrDK4pperYXekPKzuZW576uXL324IOwS4Kmw,46348
 kafka/coordinator/heartbeat.py,sha256=LeJJlwz1oUEOfEMIFT-R7ZOHBQ-b-luVKwmKyWxLfDo,3242
 kafka/coordinator/protocol.py,sha256=wTaIOnUVbj0CKXZ82FktZo-zMRvOCk3hdQAoHJ62e3I,1041
@@ -67,7 +67,7 @@ kafka/partitioner/__init__.py,sha256=Fks3C5_kokVWYw1Ad5wv0sVVzaaBtOejL-2bIL1yRII
 kafka/partitioner/default.py,sha256=tW-RC1PWIPRDEbeEAaPTLn-00oiZnXoVouEk9AnYE4w,2879
 kafka/producer/__init__.py,sha256=i3Wxih0NHjmqCkRNE54ial8fBp9siqabUE6ZGyL6oX8,122
 kafka/producer/future.py,sha256=UC3-g9QlgVFmbitrtMXVpeP0Pbvr7xl2kcw6bAehKG8,2983
-kafka/producer/kafka.py,sha256=oGO-UxoVZEFdBLOQ7zEqeDJWXMxKyUdNV-pCRU3jZmg,53302
+kafka/producer/kafka.py,sha256=T0c-QmoagY6HMyomNlvYaYSVfUHe42W72EDiegX6XyY,53573
 kafka/producer/record_accumulator.py,sha256=xNkHOCmganxDfa3W_Y3iBLT4RaAOZi0Lix-mUzsp2aQ,28170
 kafka/producer/sender.py,sha256=8-TLTw6vQO7AheWSDPI33cQdWMyTDxi1k-pkXuUb9k0,37789
 kafka/producer/transaction_manager.py,sha256=q3e9Lc9o-ofWvjT9FbHdTQH08XQBeRtoQEcQHGcnp7g,41535
@@ -120,7 +120,7 @@ kafka/vendor/enum34.py,sha256=-u-lxAiJMt6ru4Do7NUDY9OpeWkYJMksb2xengJawFE,31204
 kafka/vendor/selectors34.py,sha256=gxejLO4eXf8mRSGXaQiknPig3GdX1rtsZiYOQJVuAy8,20594
 kafka/vendor/six.py,sha256=lLBa9_HrANP5BMZ7twEzg1M3wofwPmXyptuWmHX0brY,34826
 kafka/vendor/socketpair.py,sha256=Fi3PoY1Okkppab720wFk1BhHXyjcw7hi5DwhqrYZH2Y,2737
-kafka_python-2.2.16.dist-info/METADATA,sha256=H-5YJSGk5csYCUOY1A6V8Yxboixmay0cG60di1TsGeU,9952
-kafka_python-2.2.16.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-kafka_python-2.2.16.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
-kafka_python-2.2.16.dist-info/RECORD,,
+kafka_python-2.2.17.dist-info/METADATA,sha256=u8fByYt-bY1oySuBwxpK9MiR1aGM5AFpNX5wNcTspWA,9952
+kafka_python-2.2.17.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+kafka_python-2.2.17.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
+kafka_python-2.2.17.dist-info/RECORD,,