kafka-python 2.3.0__py2.py3-none-any.whl → 2.3.1__py2.py3-none-any.whl

kafka/codec.py

@@ -327,7 +327,5 @@ def zstd_encode(payload):
 def zstd_decode(payload):
     if not zstd:
         raise NotImplementedError("Zstd codec is not available")
-    try:
-        return zstd.ZstdDecompressor().decompress(payload)
-    except zstd.ZstdError:
-        return zstd.ZstdDecompressor().decompress(payload, max_output_size=ZSTD_MAX_OUTPUT_SIZE)
+    with zstd.ZstdDecompressor().stream_reader(io.BytesIO(payload), read_across_frames=True) as reader:
+        return reader.read()

kafka/consumer/fetcher.py

@@ -250,16 +250,17 @@ class Fetcher(six.Iterator):
                 break
 
             if future.succeeded():
-                fetched_offsets.update(future.value[0])
-                if not future.value[1]:
+                offsets, retry = future.value
+                fetched_offsets.update(offsets)
+                if not retry:
                     return fetched_offsets
 
-                timestamps = {tp: timestamps[tp] for tp in future.value[1]}
+                timestamps = {tp: timestamps[tp] for tp in retry}
 
             elif not future.retriable():
                 raise future.exception  # pylint: disable-msg=raising-bad-type
 
-            if future.exception.invalid_metadata or self._client.cluster.need_update:
+            elif future.exception.invalid_metadata or self._client.cluster.need_update:
                 refresh_future = self._client.cluster.request_update()
                 self._client.poll(future=refresh_future, timeout_ms=timer.timeout_ms)
 

kafka/coordinator/consumer.py

@@ -457,7 +457,8 @@ class ConsumerCoordinator(BaseCoordinator):
             self._client.poll(future=future, timeout_ms=timer.timeout_ms)
 
             if future.is_done:
-                del self._offset_fetch_futures[future_key]
+                if future_key in self._offset_fetch_futures:
+                    del self._offset_fetch_futures[future_key]
 
                 if future.succeeded():
                     return future.value

kafka/protocol/types.py

@@ -213,6 +213,17 @@ class Array(AbstractType):
 
 
 class UnsignedVarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value = VarInt32.decode(data)
+        return (value << 1) ^ (value >> 31)
+
+    @classmethod
+    def encode(cls, value):
+        return VarInt32.encode((value >> 1) ^ -(value & 1))
+
+
+class VarInt32(AbstractType):
     @classmethod
     def decode(cls, data):
         value, i = 0, 0
@@ -225,10 +236,12 @@ class UnsignedVarInt32(AbstractType):
             if i > 28:
                 raise ValueError('Invalid value {}'.format(value))
         value |= b << i
-        return value
+        return (value >> 1) ^ -(value & 1)
 
     @classmethod
     def encode(cls, value):
+        # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 31)
         value &= 0xffffffff
         ret = b''
         while (value & 0xffffff80) != 0:
@@ -239,25 +252,12 @@ class UnsignedVarInt32(AbstractType):
         return ret
 
 
-class VarInt32(AbstractType):
-    @classmethod
-    def decode(cls, data):
-        value = UnsignedVarInt32.decode(data)
-        return (value >> 1) ^ -(value & 1)
-
-    @classmethod
-    def encode(cls, value):
-        # bring it in line with the java binary repr
-        value &= 0xffffffff
-        return UnsignedVarInt32.encode((value << 1) ^ (value >> 31))
-
-
 class VarInt64(AbstractType):
     @classmethod
     def decode(cls, data):
         value, i = 0, 0
         while True:
-            b = data.read(1)
+            b, = struct.unpack('B', data.read(1))
             if not (b & 0x80):
                 break
             value |= (b & 0x7f) << i
@@ -270,14 +270,14 @@ class VarInt64(AbstractType):
     @classmethod
     def encode(cls, value):
         # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 63)
         value &= 0xffffffffffffffff
-        v = (value << 1) ^ (value >> 63)
         ret = b''
-        while (v & 0xffffffffffffff80) != 0:
+        while (value & 0xffffffffffffff80) != 0:
             b = (value & 0x7f) | 0x80
             ret += struct.pack('B', b)
-            v >>= 7
-        ret += struct.pack('B', v)
+            value >>= 7
+        ret += struct.pack('B', value)
         return ret
 
 
@@ -322,8 +322,9 @@ class TaggedFields(AbstractType):
         for k, v in value.items():
             # do we allow for other data types ?? It could get complicated really fast
             assert isinstance(v, bytes), 'Value {} is not a byte array'.format(v)
-            assert isinstance(k, int) and k > 0, 'Key {} is not a positive integer'.format(k)
+            assert isinstance(k, int) and k >= 0, 'Key {} is not a non-negative integer'.format(k)
             ret += UnsignedVarInt32.encode(k)
+            ret += UnsignedVarInt32.encode(len(v))
             ret += v
         return ret
 

kafka/version.py

@@ -1 +1 @@
-__version__ = '2.3.0'
+__version__ = '2.3.1'

{kafka_python-2.3.0.dist-info → kafka_python-2.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.3.0
+Version: 2.3.1
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python
@@ -66,22 +66,11 @@ Python client for the Apache Kafka distributed stream processing system.
 kafka-python is designed to function much like the official java client, with a
 sprinkling of pythonic interfaces (e.g., consumer iterators).
 
-kafka-python is best used with newer brokers (0.9+), but is backwards-compatible with
-older versions (to 0.8.0). Some features will only be enabled on newer brokers.
-For example, fully coordinated consumer groups -- i.e., dynamic partition
-assignment to multiple consumers in the same group -- requires use of 0.9+ kafka
-brokers. Supporting this feature for earlier broker releases would require
-writing and maintaining custom leadership election and membership / health
-check code (perhaps using zookeeper or consul). For older brokers, you can
-achieve something similar by manually assigning different partitions to each
-consumer instance with config management tools like chef, ansible, etc. This
-approach will work fine, though it does not support rebalancing on failures.
-See https://kafka-python.readthedocs.io/en/master/compatibility.html
-for more details.
-
 Please note that the master branch may contain unreleased features. For release
 documentation, please see readthedocs and/or python's inline help.
 
+New in 2.3 release: python -m kafka.* interfaces for quick scripts and testing.
+
 .. code-block:: bash
 
     $ pip install kafka-python
@@ -232,6 +221,14 @@ for more details.
     metrics = producer.metrics()
 
 
+Module CLI Interface
+********************
+
+kafka-python also provides simple command-line interfaces for consumer, producer, and admin clients.
+Access via ``python -m kafka.consumer``, ``python -m kafka.producer``, and ``python -m kafka.admin``.
+See https://kafka-python.readthedocs.io/en/master/usage.html for more details.
+
+
 Thread safety
 *************
 

{kafka_python-2.3.0.dist-info → kafka_python-2.3.1.dist-info}/RECORD

@@ -2,14 +2,14 @@ kafka/__init__.py,sha256=4dvHKZAxmD_4tfJ5wGcRV2X78vPcm8vsUoqceULevjA,1077
 kafka/__main__.py,sha256=HNnJxekZZkNDXQwqnE5AfaM56JNCNYvq135wu4jxvhk,107
 kafka/client_async.py,sha256=NFjHZu-aa4tUKOz-l9d9-dkN8dKt9OEJ-4ljxNgbw-M,57031
 kafka/cluster.py,sha256=kTWqLHD-CZY_ua2LftVSvEjmNAL7KJ98Rd17zA177FU,16915
-kafka/codec.py,sha256=8NZpnehzNrhSBIjzbPVSvyFbSeLAqEntE7BfVHu-_9I,10036
+kafka/codec.py,sha256=_jglwQ8yn-B85hzf0nvOvazRwGQSBx22Bb8R4sSCvcs,9977
 kafka/conn.py,sha256=hDwKQ93zpZAWfACdiInEfcBSD6MpYCxVNoAjFneQW9Q,70406
 kafka/errors.py,sha256=6DbK-_ov_TROE_q8PlaxuTPh9LVnguwEIBcNBiakW6k,33451
 kafka/future.py,sha256=ZQStbfUYIPJRrgMfAWxxjrIRVxsw4WCtSR0J0bkyGno,2847
 kafka/socks5_wrapper.py,sha256=h0Gag3xAOp8io2MfzeYLOiNvLRmq3rkmpZ6Aj-9uKTw,10716
 kafka/structs.py,sha256=SJGzmLdV21jZyQ7247k0WFy16UiusgTHK3I-e4qzI-E,3058
 kafka/util.py,sha256=ncFqg0mXda_ipyLoAw_EF4dX8-lYanC2QtzC-l_F7-w,4585
-kafka/version.py,sha256=HkuhK2leBUgujkTPPE8VZAlHBDxmueEzVb4sz0_wlyQ,22
+kafka/version.py,sha256=0Ns5BhVD3XOEWhtSUJ76K319NapPiQfZKsWZRt-kZOk,22
 kafka/admin/__init__.py,sha256=S_XxqyyV480_yXhttK79XZqNAmZyXRjspd3SoqYykE8,720
 kafka/admin/__main__.py,sha256=mQrmllHNZx4SnFWd5WHJdQqfsqunFR1pnkIVppm3MqU,109
 kafka/admin/acl_resource.py,sha256=UVfKDMsxRl-yGeCrbn0OX74aDbBQTibLgWMfCxkle5I,8480
@@ -46,12 +46,12 @@ kafka/cli/consumer/__init__.py,sha256=Kw4v9vSLBXD2KxSSMrloAuauWi7Zke2C2M8PCvcGJ4
 kafka/cli/producer/__init__.py,sha256=Wy_8p1TSzDkd9YnFvgG91AFXN6j8toVVfrNA0r_kplY,2633
 kafka/consumer/__init__.py,sha256=NDdvtyuJgFyQZahqL9i5sYXGP6rOMIXWwHQEaZ1fCcs,122
 kafka/consumer/__main__.py,sha256=JcySmP6wWi5NxK4icqMWPbFkf2722X4J-gQvbQZtDsk,112
-kafka/consumer/fetcher.py,sha256=csJ_uMFS3vVE5J6VxamxVJd0pnTsMHLokXY4ZI42Py0,69355
+kafka/consumer/fetcher.py,sha256=TR6LJ2pa0LfJ5kTC-6DPHkMPO2CAo8qV-iO79VQ5F10,69375
 kafka/consumer/group.py,sha256=iMKfNEjCzL6UGbwA7MWBU_o1gaLRCIxFSj-JmCMF3RM,60637
 kafka/consumer/subscription_state.py,sha256=bK-YTVbOzhy8OB206QAfXVuo7zPA9YqYXnrRRST369c,24289
 kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kafka/coordinator/base.py,sha256=QP0YwJVj244BPwx9vP5XlI-E6geniztsJstiyglZI2k,58502
-kafka/coordinator/consumer.py,sha256=OUiA1KXaBamK3Bbdx-SuHrHceC31KYesp2sT5Yt1mxE,48509
+kafka/coordinator/consumer.py,sha256=ZHiiCKRsglPwhgQ5u7QPeFmwKDCRaJbA05ioZJITzWI,48574
 kafka/coordinator/heartbeat.py,sha256=LeJJlwz1oUEOfEMIFT-R7ZOHBQ-b-luVKwmKyWxLfDo,3242
 kafka/coordinator/protocol.py,sha256=Jq5OvRGGSrVCymufXsD-ZfUK-3ntsuFZE3Zu7lIaPkg,1041
 kafka/coordinator/subscription.py,sha256=Rzdl2WnZdFB6v9M1cpZiWB_yzmVUJzB2X15KttMdBmw,922
@@ -122,7 +122,7 @@ kafka/protocol/sasl_authenticate.py,sha256=HaFAHPRhCKgyGEoJ5LwGffcpMUBNCphgBgXCs
 kafka/protocol/sasl_handshake.py,sha256=WzQh9HBRemXvShrczkN4rd4SM-hNdes1khMzPRvcRQQ,982
 kafka/protocol/struct.py,sha256=iHVlM7QySnVDBbN_v4mRFmU6LJ6kslvITRdaYIsus-Q,2377
 kafka/protocol/txn_offset_commit.py,sha256=_6Wr-SabUd9q09Tj9oG43AVZcqlW3LYbqXNW1Pvk9vs,2250
-kafka/protocol/types.py,sha256=ovkiTrlIH1r4sm7fO0Ovme2lVNaxX3Vlli7phneBSek,11121
+kafka/protocol/types.py,sha256=SXAlAfrHi5q5siMxYxiMcWN_eX0fjcQs7ZRphpuUT-A,11237
 kafka/record/__init__.py,sha256=Q20hP_R5XX3AEnAlPkpoWzTLShESvxUT2OLXmI-JYEQ,129
 kafka/record/_crc32c.py,sha256=Ok-P62Yvg6D6rMGM9Z56OMjZWQlnps4xBbakg-sdxvI,5761
 kafka/record/abc.py,sha256=z1UYURHbD2RyyGRpVXKP598jck5eXU9p4M6iUo6ZSFo,4110
@@ -145,7 +145,7 @@ kafka/vendor/enum34.py,sha256=-u-lxAiJMt6ru4Do7NUDY9OpeWkYJMksb2xengJawFE,31204
 kafka/vendor/selectors34.py,sha256=gxejLO4eXf8mRSGXaQiknPig3GdX1rtsZiYOQJVuAy8,20594
 kafka/vendor/six.py,sha256=lLBa9_HrANP5BMZ7twEzg1M3wofwPmXyptuWmHX0brY,34826
 kafka/vendor/socketpair.py,sha256=Fi3PoY1Okkppab720wFk1BhHXyjcw7hi5DwhqrYZH2Y,2737
-kafka_python-2.3.0.dist-info/METADATA,sha256=SrwjFkP48zwY87FMT6Avd7zRMg9N_hjaCB-rbx8wlpA,9951
-kafka_python-2.3.0.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-kafka_python-2.3.0.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
-kafka_python-2.3.0.dist-info/RECORD,,
+kafka_python-2.3.1.dist-info/METADATA,sha256=9CuWXkHGkkSqO8pfMQSle7IvLXZ250msWSY2d7EqIik,9494
+kafka_python-2.3.1.dist-info/WHEEL,sha256=TdQ5LtNwLuxTCjgxN51AgdU5w-KkB9ttmLbzjTH02pg,109
+kafka_python-2.3.1.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
+kafka_python-2.3.1.dist-info/RECORD,,

{kafka_python-2.3.0.dist-info → kafka_python-2.3.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (82.0.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any