talos-python3 3.1.3__tar.gz → 3.1.4__tar.gz

{talos_python3-3.1.3 → talos_python3-3.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: talos_python3
-Version: 3.1.3
+Version: 3.1.4
 Summary: talos python3 sdk
 Home-page: UNKNOWN
 Author: huyumei

{talos_python3-3.1.3 → talos_python3-3.1.4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name='talos_python3',
-    version='3.1.3',
+    version='3.1.4',
     author='huyumei',
     author_email='huyumei@xiaomi.com',
     maintainer='fangchengjin',
@@ -13,7 +13,9 @@ setup(
     zip_safe=False,
     install_requires=[
         'cffi',
-        'python-snappy==0.6.1',
+        'python-snappy==0.6.1; python_version < "3.8"',
+        'python-snappy; python_version >= "3.8"',
+        'zstandard==0.21.0',
         'atomic',
         'dnspython',
         'requests',

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos/client/TalosClientConfig.py

@@ -190,7 +190,7 @@ class TalosClientConfig:
             TalosProducerConfigKeys.GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE_DEFAULT)
 
         if not (self._compressionType == "NONE" or self._compressionType == "GZIP"
-                or self._compressionType == "SNAPPY"):
+                or self._compressionType == "SNAPPY" or self._compressionType == "ZSTD"):
             raise RuntimeError("Unsupported Compression Type: " + self._compressionType)
 
         # consumer configs

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos/client/compression/Compression.py

@@ -16,6 +16,7 @@ from talos.client.serialization.MessageSerialization import MessageSerialization
 from talos.client.Constants import Constants
 from talos.client.compression.Gzip import Gzip
 from talos.client.compression.Snappy import Snappy
+from talos.client.compression.Zstd import Zstd
 from io import BytesIO
 import logging
 import traceback
@@ -26,6 +27,7 @@ class Compression(object):
     logger = logging.getLogger("Compression")
     gzip = Gzip()
     snappy = Snappy()
+    zstd = Zstd()
 
     def compress(self, msgList=None, compressionType=None):
         return self.do_compress(msgList, compressionType, MessageSerializationFactory().get_default_message_version())
@@ -59,6 +61,8 @@ class Compression(object):
                 messageBlockData = self.gzip.compress(dataOutputIo.getvalue())
             elif messageCompressionType == MessageCompressionType.SNAPPY:
                 messageBlockData = self.snappy.encode(dataOutputIo.getvalue())
+            elif messageCompressionType == MessageCompressionType.ZSTD:
+                messageBlockData = self.zstd.compress(dataOutputIo.getvalue())
             else:
                 raise RuntimeError("Unsupported Compression Type: " + str(compressionType))
 
@@ -97,6 +101,8 @@ class Compression(object):
             messageBlockData = BytesIO(self.gzip.decompress(messageBlock.messageBlock))
         elif messageBlock.compressionType == MessageCompressionType.SNAPPY:
             messageBlockData = BytesIO(self.snappy.decode(messageBlock.messageBlock))
+        elif messageBlock.compressionType == MessageCompressionType.ZSTD:
+            messageBlockData = BytesIO(self.zstd.decompress(messageBlock.messageBlock))
 
         messageNumber = messageBlock.messageNumber
 

talos_python3-3.1.4/talos/client/compression/Zstd.py

@@ -0,0 +1,25 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright 2026, Xiaomi.
+# All rights reserved.
+#
+
+class Zstd:
+    def __init__(self):
+        try:
+            import zstandard
+        except ImportError:
+            raise ImportError(
+                "zstandard is required for ZSTD compression. "
+                "Please install: pip install zstandard==0.21.0"
+            )
+
+        self._zstd = zstandard
+        self._compressor = zstandard.ZstdCompressor()
+        self._decompressor = zstandard.ZstdDecompressor()
+
+    def compress(self, c_data):
+        return self._compressor.compress(c_data)
+
+    def decompress(self, c_data):
+        return self._decompressor.decompress(c_data)

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos/consumer/TalosConsumer.py

@@ -180,7 +180,7 @@ class TalosConsumer(object):
             return
 
         topicPartitionNum = response.partitionNumber
-        if self.partitionNumber < topicPartitionNum:
+        if self.partitionNumber != topicPartitionNum:
             logger.info("partitionNumber changed from " + str(self.partitionNumber) + " to " +
                         str(topicPartitionNum) + ", execute a re-balance task.")
             # update partition number and call the re - balance
@@ -458,6 +458,14 @@ class TalosConsumer(object):
                 hasList = self.get_has_list()
                 has = len(hasList)
 
+                hasList.sort()
+                # release all reduced partition and wait next cycle to balance
+                if len(hasList) > 0 and hasList[len(hasList) - 1] >= self.partitionNumber:
+                    for partitionId in hasList:
+                        if partitionId >= self.partitionNumber:
+                            toReleaseList.append(partitionId)
+                    break
+
                 # workerNum > partitionNum, idle workers have no match target, do nothing
                 if i >= len(targetList):
                     break

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos/producer/TalosProducerConfig.py

@@ -54,7 +54,7 @@ class TalosProducerConfig(TalosClientConfig):
             TalosProducerConfigKeys.GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE_DEFAULT)
 
         if not (self._compressionType == "NONE" or self._compressionType == "GZIP"
-                or self._compressionType == "SNAPPY"):
+                or self._compressionType == "SNAPPY" or self._compressionType == "ZSTD"):
             raise RuntimeError("Unsupported Compression Type: " + self._compressionType)
 
     def parameter_checking(self):

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos/producer/TalosProducerConfigKey.py

@@ -84,7 +84,7 @@ class TalosProducerConfigKeys(TalosClientConfigKeys):
     GALAXY_TALOS_PRODUCER_WAIT_PARTITION_WORKING_TIME = "galaxy.talos.producer.wait.partition.working.time.milli"
     GALAXY_TALOS_PRODUCER_WAIT_PARTITION_WORKING_TIME_DEFAULT = 0.2
 
-    # The producer compression type, right now suport "NONE", "SNAPPY" and "GZIP";
+    # The producer compression type, right now support "NONE", "SNAPPY", "GZIP" and "ZSTD";
     # default is "SNAPPY";
     GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE = "galaxy.talos.producer.compression.type"
     GALAXY_TALOS_PRODUCER_COMPRESSION_TYPE_DEFAULT = "SNAPPY"

talos_python3-3.1.4/talos/test/client/test_zstd_compression.py

@@ -0,0 +1,23 @@
+# Copyright 2026, Xiaomi.
+# All rights reserved.
+
+import unittest
+
+from talos.client.compression.Compression import Compression
+from talos.thrift.message.ttypes import Message
+
+
+class test_ZstdCompression(unittest.TestCase):
+
+    def test_compress_decompress_roundtrip(self):
+        msg1 = Message(partitionKey="k1", sequenceNumber="s1", message=b"hello", createTimestamp=1)
+        msg2 = Message(partitionKey="k2", sequenceNumber="s2", message=b"world", createTimestamp=2)
+
+        msg_block = Compression().compress([msg1, msg2], "ZSTD")
+        self.assertIsNotNone(msg_block)
+
+        out = Compression().decompress([msg_block], 0)
+        self.assertEqual(2, len(out))
+
+        self.assertEqual(b"hello", out[0].message.message)
+        self.assertEqual(b"world", out[1].message.message)

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos/thrift/message/ttypes.py

@@ -58,17 +58,23 @@ class MessageCompressionType:
   NONE = 0
   SNAPPY = 1
   GZIP = 2
+  ZSTD = 3
+  LZ4 = 4
 
   _VALUES_TO_NAMES = {
     0: "NONE",
     1: "SNAPPY",
     2: "GZIP",
+    3: "ZSTD",
+    4: "LZ4",
   }
 
   _NAMES_TO_VALUES = {
     "NONE": 0,
     "SNAPPY": 1,
     "GZIP": 2,
+    "ZSTD": 3,
+    "LZ4": 4,
   }
 
 

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos_python3.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: talos-python3
-Version: 3.1.3
+Version: 3.1.4
 Summary: talos python3 sdk
 Home-page: UNKNOWN
 Author: huyumei

{talos_python3-3.1.3 → talos_python3-3.1.4}/talos_python3.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ talos/client/__init__.py
 talos/client/compression/Compression.py
 talos/client/compression/Gzip.py
 talos/client/compression/Snappy.py
+talos/client/compression/Zstd.py
 talos/client/compression/__init__.py
 talos/client/serialization/MessageSerialization.py
 talos/client/serialization/MessageSerializationFactory.py
@@ -41,7 +42,6 @@ talos/example/TalosAdminDemo.py
 talos/example/TalosConsumerDemo.py
 talos/example/TalosProducerDemo.py
 talos/example/__init__.py
-talos/example/test.py
 talos/producer/BufferedMessageCount.py
 talos/producer/PartitionMessageQueue.py
 talos/producer/PartitionSender.py
@@ -61,6 +61,7 @@ talos/test/test.py
 talos/test/admin/__init__.py
 talos/test/client/__init__.py
 talos/test/client/test_socket.py
+talos/test/client/test_zstd_compression.py
 talos/test/consumer/__init__.py
 talos/test/consumer/test_PartitionFetcher.py
 talos/test/consumer/test_SimpleConsumer.py

talos_python3-3.1.4/talos_python3.egg-info/requires.txt

@@ -0,0 +1,12 @@
+IPy
+atomic
+cffi
+dnspython
+requests
+zstandard==0.21.0
+
+[:python_version < "3.8"]
+python-snappy==0.6.1
+
+[:python_version >= "3.8"]
+python-snappy

talos_python3-3.1.3/talos/example/test.py

@@ -1,46 +0,0 @@
-# Talos Consumer 封装类
-from talos.client.TalosClientConfig import TalosClientConfig
-from talos.example.TalosConsumerDemo import MyMessageProcessorFactory
-from talos.thrift.auth.ttypes import Credential
-from talos.thrift.topic.ttypes import TopicAndPartition
-from talos.consumer.TalosConsumer import TalosConsumer
-from talos.consumer.MessageProcessor import MessageProcessor
-from talos.consumer.MessageProcessorFactory import MessageProcessorFactory
-from talos.thrift.auth.ttypes import UserType
-from atomic import AtomicLong
-import logging
-import traceback
-
-class TalosConsumerWrapper:
-    accessKey = "AKM5DDIDEBDWBBD2PV"
-    accessSecret = "w8nium9H1/1milUJkHAVGhpiDRchLxnd4ZAUbdE6"
-    consumerGroup = "ConsumerGroup-fcj"
-    clientPrefix = "ClientPrefix"
-    pro = {
-        "galaxy.talos.service.endpoint": "http://ap-tjv1autopilotsrv-talos.api.xiaomi.net",
-        "galaxy.talos.client.falcon.monitor.switch": False
-    }
-    count = 1
-
-    def __init__(self):
-        self.consumerConfig = TalosClientConfig(self.pro)
-        self.credential = Credential(UserType.DEV_XIAOMI, self.accessKey, self.accessSecret)
-        self.talosConsumer = None
-
-    def start(self, topic_name, partitionCheckPoint=None):
-        """
-        启动消费者。
-        """
-        # logger.info(f"start consumer: service_manager={service_manager}, factory={factory}, topic_name={topic_name}")
-        self.talosConsumer = TalosConsumer(
-            consumerGroup=self.consumerGroup,
-            consumerConfig=self.consumerConfig,
-            credential=self.credential,
-            topicName=topic_name,
-            messageProcessorFactory= MyMessageProcessorFactory(),
-            clientPrefix=self.clientPrefix
-        )
-        return self.talosConsumer
-
-consumerDemo = TalosConsumerWrapper()
-consumerDemo.start("geo_result_receive_preview")

talos_python3-3.1.3/talos_python3.egg-info/requires.txt

@@ -1,6 +0,0 @@
-cffi
-python-snappy==0.6.1
-atomic
-dnspython
-requests
-IPy