talos-python3 3.1.2__tar.gz → 3.1.4__tar.gz

{talos-python3-3.1.2/talos_python3.egg-info → talos_python3-3.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: talos-python3
-Version: 3.1.2
+Name: talos_python3
+Version: 3.1.4
 Summary: talos python3 sdk
 Home-page: UNKNOWN
 Author: huyumei

{talos-python3-3.1.2 → talos_python3-3.1.4}/setup.py

@@ -1,8 +1,8 @@
 from setuptools import find_packages, setup
 
 setup(
-    name='talos-python3',
-    version='3.1.2',
+    name='talos_python3',
+    version='3.1.4',
     author='huyumei',
     author_email='huyumei@xiaomi.com',
     maintainer='fangchengjin',
@@ -13,7 +13,9 @@ setup(
     zip_safe=False,
     install_requires=[
         'cffi',
-        'python-snappy==0.6.1',
+        'python-snappy==0.6.1; python_version < "3.8"',
+        'python-snappy; python_version >= "3.8"',
+        'zstandard==0.21.0',
         'atomic',
         'dnspython',
         'requests',

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/client/TalosClientConfig.py

@@ -190,7 +190,7 @@ class TalosClientConfig:
             TalosProducerConfigKeys.GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE_DEFAULT)
 
         if not (self._compressionType == "NONE" or self._compressionType == "GZIP"
-                or self._compressionType == "SNAPPY"):
+                or self._compressionType == "SNAPPY" or self._compressionType == "ZSTD"):
             raise RuntimeError("Unsupported Compression Type: " + self._compressionType)
 
         # consumer configs

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/client/compression/Compression.py

@@ -16,6 +16,7 @@ from talos.client.serialization.MessageSerialization import MessageSerialization
 from talos.client.Constants import Constants
 from talos.client.compression.Gzip import Gzip
 from talos.client.compression.Snappy import Snappy
+from talos.client.compression.Zstd import Zstd
 from io import BytesIO
 import logging
 import traceback
@@ -26,6 +27,7 @@ class Compression(object):
     logger = logging.getLogger("Compression")
     gzip = Gzip()
     snappy = Snappy()
+    zstd = Zstd()
 
     def compress(self, msgList=None, compressionType=None):
         return self.do_compress(msgList, compressionType, MessageSerializationFactory().get_default_message_version())
@@ -59,6 +61,8 @@ class Compression(object):
                 messageBlockData = self.gzip.compress(dataOutputIo.getvalue())
             elif messageCompressionType == MessageCompressionType.SNAPPY:
                 messageBlockData = self.snappy.encode(dataOutputIo.getvalue())
+            elif messageCompressionType == MessageCompressionType.ZSTD:
+                messageBlockData = self.zstd.compress(dataOutputIo.getvalue())
             else:
                 raise RuntimeError("Unsupported Compression Type: " + str(compressionType))
 
@@ -97,6 +101,8 @@ class Compression(object):
             messageBlockData = BytesIO(self.gzip.decompress(messageBlock.messageBlock))
         elif messageBlock.compressionType == MessageCompressionType.SNAPPY:
             messageBlockData = BytesIO(self.snappy.decode(messageBlock.messageBlock))
+        elif messageBlock.compressionType == MessageCompressionType.ZSTD:
+            messageBlockData = BytesIO(self.zstd.decompress(messageBlock.messageBlock))
 
         messageNumber = messageBlock.messageNumber
 

talos_python3-3.1.4/talos/client/compression/Zstd.py

@@ -0,0 +1,25 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright 2026, Xiaomi.
+# All rights reserved.
+#
+
+class Zstd:
+    def __init__(self):
+        try:
+            import zstandard
+        except ImportError:
+            raise ImportError(
+                "zstandard is required for ZSTD compression. "
+                "Please install: pip install zstandard==0.21.0"
+            )
+
+        self._zstd = zstandard
+        self._compressor = zstandard.ZstdCompressor()
+        self._decompressor = zstandard.ZstdDecompressor()
+
+    def compress(self, c_data):
+        return self._compressor.compress(c_data)
+
+    def decompress(self, c_data):
+        return self._decompressor.decompress(c_data)

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/consumer/TalosConsumer.py

@@ -180,7 +180,7 @@ class TalosConsumer(object):
             return
 
         topicPartitionNum = response.partitionNumber
-        if self.partitionNumber < topicPartitionNum:
+        if self.partitionNumber != topicPartitionNum:
             logger.info("partitionNumber changed from " + str(self.partitionNumber) + " to " +
                         str(topicPartitionNum) + ", execute a re-balance task.")
             # update partition number and call the re - balance
@@ -458,6 +458,14 @@ class TalosConsumer(object):
                 hasList = self.get_has_list()
                 has = len(hasList)
 
+                hasList.sort()
+                # release all reduced partition and wait next cycle to balance
+                if len(hasList) > 0 and hasList[len(hasList) - 1] >= self.partitionNumber:
+                    for partitionId in hasList:
+                        if partitionId >= self.partitionNumber:
+                            toReleaseList.append(partitionId)
+                    break
+
                 # workerNum > partitionNum, idle workers have no match target, do nothing
                 if i >= len(targetList):
                     break

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/producer/TalosProducer.py

@@ -261,10 +261,8 @@ class TalosProducer(object):
                                                     self.updatePartitionIdMsgNumber)
 
     def check_message_partition_key_validity(self, partitionKey=None):
-        try:
-            assert not partitionKey
-        except AssertionError as e:
-            raise e
+        if not partitionKey:
+            raise InvalidArgumentError("Field \"partitionKey\" must be set")
 
         if len(partitionKey) < self.partitionKeyMinLen or len(partitionKey) > self.partitionKeyMaxLen:
             raise InvalidArgumentError("Invalid partition key which length must be at "

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/producer/TalosProducerConfig.py

@@ -54,7 +54,7 @@ class TalosProducerConfig(TalosClientConfig):
             TalosProducerConfigKeys.GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE_DEFAULT)
 
         if not (self._compressionType == "NONE" or self._compressionType == "GZIP"
-                or self._compressionType == "SNAPPY"):
+                or self._compressionType == "SNAPPY" or self._compressionType == "ZSTD"):
             raise RuntimeError("Unsupported Compression Type: " + self._compressionType)
 
     def parameter_checking(self):

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/producer/TalosProducerConfigKey.py

@@ -84,7 +84,7 @@ class TalosProducerConfigKeys(TalosClientConfigKeys):
     GALAXY_TALOS_PRODUCER_WAIT_PARTITION_WORKING_TIME = "galaxy.talos.producer.wait.partition.working.time.milli"
     GALAXY_TALOS_PRODUCER_WAIT_PARTITION_WORKING_TIME_DEFAULT = 0.2
 
-    # The producer compression type, right now suport "NONE", "SNAPPY" and "GZIP";
+    # The producer compression type, right now support "NONE", "SNAPPY", "GZIP" and "ZSTD";
     # default is "SNAPPY";
     GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE = "galaxy.talos.producer.compression.type"
     GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE_DEFAULT = "SNAPPY"

talos_python3-3.1.4/talos/test/client/test_zstd_compression.py

@@ -0,0 +1,23 @@
+# Copyright 2026, Xiaomi.
+# All rights reserved.
+
+import unittest
+
+from talos.client.compression.Compression import Compression
+from talos.thrift.message.ttypes import Message
+
+
+class test_ZstdCompression(unittest.TestCase):
+
+    def test_compress_decompress_roundtrip(self):
+        msg1 = Message(partitionKey="k1", sequenceNumber="s1", message=b"hello", createTimestamp=1)
+        msg2 = Message(partitionKey="k2", sequenceNumber="s2", message=b"world", createTimestamp=2)
+
+        msg_block = Compression().compress([msg1, msg2], "ZSTD")
+        self.assertIsNotNone(msg_block)
+
+        out = Compression().decompress([msg_block], 0)
+        self.assertEqual(2, len(out))
+
+        self.assertEqual(b"hello", out[0].message.message)
+        self.assertEqual(b"world", out[1].message.message)

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos/thrift/message/ttypes.py

@@ -58,17 +58,23 @@ class MessageCompressionType:
   NONE = 0
   SNAPPY = 1
   GZIP = 2
+  ZSTD = 3
+  LZ4 = 4
 
   _VALUES_TO_NAMES = {
     0: "NONE",
     1: "SNAPPY",
     2: "GZIP",
+    3: "ZSTD",
+    4: "LZ4",
   }
 
   _NAMES_TO_VALUES = {
     "NONE": 0,
     "SNAPPY": 1,
     "GZIP": 2,
+    "ZSTD": 3,
+    "LZ4": 4,
   }
 
 

{talos-python3-3.1.2 → talos_python3-3.1.4/talos_python3.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: talos-python3
-Version: 3.1.2
+Version: 3.1.4
 Summary: talos python3 sdk
 Home-page: UNKNOWN
 Author: huyumei

{talos-python3-3.1.2 → talos_python3-3.1.4}/talos_python3.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ talos/client/__init__.py
 talos/client/compression/Compression.py
 talos/client/compression/Gzip.py
 talos/client/compression/Snappy.py
+talos/client/compression/Zstd.py
 talos/client/compression/__init__.py
 talos/client/serialization/MessageSerialization.py
 talos/client/serialization/MessageSerializationFactory.py
@@ -60,6 +61,7 @@ talos/test/test.py
 talos/test/admin/__init__.py
 talos/test/client/__init__.py
 talos/test/client/test_socket.py
+talos/test/client/test_zstd_compression.py
 talos/test/consumer/__init__.py
 talos/test/consumer/test_PartitionFetcher.py
 talos/test/consumer/test_SimpleConsumer.py

talos_python3-3.1.4/talos_python3.egg-info/requires.txt

@@ -0,0 +1,12 @@
+IPy
+atomic
+cffi
+dnspython
+requests
+zstandard==0.21.0
+
+[:python_version < "3.8"]
+python-snappy==0.6.1
+
+[:python_version >= "3.8"]
+python-snappy

talos-python3-3.1.2/talos_python3.egg-info/requires.txt

@@ -1,6 +0,0 @@
-cffi
-python-snappy==0.6.1
-atomic
-dnspython
-requests
-IPy