aliyun-log-python-sdk 0.9.22__tar.gz → 0.9.23__tar.gz

{aliyun_log_python_sdk-0.9.22/aliyun_log_python_sdk.egg-info → aliyun_log_python_sdk-0.9.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.22
+Version: 0.9.23
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23}/aliyun/log/compress.py

@@ -20,10 +20,19 @@ def lz_compresss(data):
 
 def zstd_compress(data):
     if not zstd_available:
-        raise LogException('UnsupportedompressType',
-                           'Unsupported compress type zstd, zstd not installed')
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type zstd, library zstd not installed')
     return zstd.compress(data, 1)
 
+def zstd_decompress(raw_size, data):
+    if not zstd_available:
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type zstd, library zstd not installed')
+    result = zstd.decompress(data)
+    if len(result) != raw_size:
+        raise LogException('DecompressError', 'uncompressed size ' + str(len(result)) + ' does not match x-log-bodyrawsize ' + str(raw_size))
+    return result
+
 class CompressType(Enum):
     UNCOMPRESSED = 1
     LZ4 = 2
@@ -62,8 +71,8 @@ class Compressor():
         if compress_type == CompressType.UNCOMPRESSED:
             return data
 
-        raise LogException('UnknownCompressType',
-                           'Unknown compress type: ' + compress_type)
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type: ' + compress_type)
 
     @staticmethod
     def decompress(data, raw_size, compress_type):
@@ -73,9 +82,12 @@ class Compressor():
 
         if compress_type == CompressType.UNCOMPRESSED:
             return data
+        
+        if compress_type == CompressType.ZSTD:
+            return zstd_decompress(raw_size, data)
 
-        raise LogException('UnknownCompressType',
-                           'Unknown compress type: ' + compress_type)
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type: ' + compress_type)
 
     @staticmethod
     def decompress_response(header, response):
@@ -93,8 +105,11 @@ class Compressor():
         if compress_type_str == 'lz4':
             return Compressor.decompress(response, raw_size, CompressType.LZ4)
 
+        if compress_type_str == 'zstd':
+            return Compressor.decompress(response, raw_size, CompressType.ZSTD)
+
         if compress_type_str in ('gzip', 'deflate'):
             return zlib.decompress(response)
 
-        raise LogException('UnknownCompressType', 'Unknown compress type: ' +
+        raise LogException('UnsupportedCompressType', 'Unsupported compress type: ' +
                            compress_type_str, resp_header=header, resp_body=response)

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23}/aliyun/log/consumer/config.py

@@ -25,7 +25,8 @@ class LogHubConfig(object):
                  cursor_position=None, heartbeat_interval=None, data_fetch_interval=None, in_order=False,
                  cursor_start_time=None, security_token=None, max_fetch_log_group_size=None, worker_pool_size=None, shared_executor=None,
                  cursor_end_time=None, credentials_refresher=None,
-                 auth_version=AUTH_VERSION_1, region='', query=None):
+                 auth_version=AUTH_VERSION_1, region='', query=None,
+                 accept_compress_type=None):
         """
 
         :param endpoint:
@@ -48,6 +49,10 @@ class LogHubConfig(object):
         :param auth_version: only support AUTH_VERSION_1 and AUTH_VERSION_4
         :param region: region of project
         :param query: the SPL query, such as *| where a = 'xxx'
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
+
         """
         self.endpoint = endpoint
         self.accessKeyId = access_key_id
@@ -71,3 +76,4 @@ class LogHubConfig(object):
         self.auth_version = auth_version
         self.region = region
         self.query = query
+        self.accept_compress_type = accept_compress_type

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23}/aliyun/log/consumer/consumer_client.py

@@ -22,18 +22,60 @@ class ConsumerClientLoggerAdapter(logging.LoggerAdapter):
 class ConsumerClient(object):
     def __init__(self, endpoint, access_key_id, access_key, project,
                  logstore, consumer_group, consumer, security_token=None, credentials_refresher=None,
-                 auth_version=AUTH_VERSION_1, region=''):
+                 auth_version=AUTH_VERSION_1, region='',
+                 accept_compress_type=None):
+        '''
+        :type endpoint: string
+        :param endpoint: the endpoint of sls project
 
+        :type access_key_id: string
+        :param access_key_id: your access key id
+
+        :type access_key: string
+        :param access_key: your access key secret
+
+        :type project: string
+        :param project: sls project
+
+        :type logstore: string
+        :param logstore: sls logstore to consume
+
+        :type consumer_group: string
+        :param consumer_group: consumer group name, the consumer group will be created when needed
+
+        :type consumer: string
+        :param consumer: consumer id, the id must be unique in the same consumer group
+
+        :type security_token: string
+        :param security_token: security token
+
+        :type credentials_refresher: CredentialsProvider
+        :param credentials_refresher: provides temporarily accessKeyId & accessKey dynamically
+
+        :type auth_version: string
+        :param auth_version: the auth version for signature, eg. AUTH_VERSION_1, AUTH_VERSION_4
+
+        :type region: string
+        :param region: the region of your project, required when auth_version is AUTH_VERSION_4
+
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
+
+        '''
         from .. import LogClient
 
-        self.mclient = LogClient(endpoint, access_key_id, access_key, security_token, auth_version=auth_version, region=region)
-        self.mclient.set_user_agent('%s-consumergroup-%s-%s' % (USER_AGENT, consumer_group, consumer))
+        self.mclient = LogClient(endpoint, access_key_id, access_key,
+                                 security_token, auth_version=auth_version, region=region)
+        self.mclient.set_user_agent(
+            '%s-consumergroup-%s-%s' % (USER_AGENT, consumer_group, consumer))
         if credentials_refresher is not None:
             self.mclient.set_credentials_auto_refresher(credentials_refresher)
         self.mproject = project
         self.mlogstore = logstore
         self.mconsumer_group = consumer_group
         self.mconsumer = consumer
+        self.accept_compress_type = accept_compress_type
         self.logger = ConsumerClientLoggerAdapter(
             logging.getLogger(__name__), {"consumer_client": self})
 
@@ -72,8 +114,8 @@ class ConsumerClient(object):
         except LogException as _:
             return None, True
 
-
     # fallback, for backward compatibility
+
     def _ensure_consumer_group_created_legacy(self, timeout, in_order):
         try:
             self.mclient.create_consumer_group(self.mproject, self.mlogstore, self.mconsumer_group,
@@ -138,4 +180,4 @@ class ConsumerClient(object):
         return self.mclient.get_end_cursor(self.mproject, self.mlogstore, shard_id).get_cursor()
 
     def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None):
-        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor,query=query)
+        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor, query=query, accept_compress_type=self.accept_compress_type)

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23}/aliyun/log/consumer/worker.py

@@ -36,7 +36,8 @@ class ConsumerWorker(Thread):
                            consumer_option.project, consumer_option.logstore, consumer_option.consumer_group_name,
                            consumer_option.consumer_name, consumer_option.securityToken,
                            credentials_refresher=consumer_option.credentials_refresher,
-                           auth_version=consumer_option.auth_version, region=consumer_option.region)
+                           auth_version=consumer_option.auth_version, region=consumer_option.region,
+                           accept_compress_type=consumer_option.accept_compress_type)
         self.shut_down_flag = False
         self.logger = ConsumerWorkerLoggerAdapter(
             logging.getLogger(__name__), {"consumer_worker": self})

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23}/aliyun/log/logclient.py

@@ -1094,7 +1094,7 @@ class LogClient(object):
         """
         return self.get_cursor(project_name, logstore_name, shard_id, "end")
 
-    def pull_logs(self, project_name, logstore_name, shard_id, cursor, count=None, end_cursor=None, compress=None, query=None):
+    def pull_logs(self, project_name, logstore_name, shard_id, cursor, count=None, end_cursor=None, compress=None, query=None, accept_compress_type=None):
         """ batch pull log data from log service
         Unsuccessful operation will cause an LogException.
 
@@ -1121,6 +1121,10 @@ class LogClient(object):
 
         :type query: string
         :param query: the SPL query, such as *| where a = 'xxx'
+        
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
 
         :return: PullLogResponse
 
@@ -1131,7 +1135,10 @@ class LogClient(object):
 
         need_compress = compress is None or compress
         if need_compress:
-            headers['Accept-Encoding'] = str(CompressType.default_compress_type())
+            if accept_compress_type is None or accept_compress_type == '':
+                headers['Accept-Encoding'] = str(CompressType.default_compress_type())
+            else:
+                headers['Accept-Encoding'] = accept_compress_type
         else:
             headers['Accept-Encoding'] = ''
 
@@ -1158,7 +1165,7 @@ class LogClient(object):
         raw_data = Compressor.decompress_response(header, resp)
         return PullLogResponse(raw_data, header)
 
-    def pull_log(self, project_name, logstore_name, shard_id, from_time, to_time, batch_size=None, compress=None, query=None):
+    def pull_log(self, project_name, logstore_name, shard_id, from_time, to_time, batch_size=None, compress=None, query=None, accept_compress_type=None):
         """ batch pull log data from log service using time-range
         Unsuccessful operation will cause an LogException. the time parameter means the time when server receives the logs
 
@@ -1185,6 +1192,10 @@ class LogClient(object):
 
         :type query: string
         :param query: the SPL query, such as *| where a = 'xxx'
+        
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
 
         :return: PullLogResponse
 
@@ -1195,7 +1206,8 @@ class LogClient(object):
 
         while True:
             res = self.pull_logs(project_name, logstore_name, shard_id, begin_cursor,
-                                 count=batch_size, end_cursor=end_cursor, compress=compress, query=query)
+                                 count=batch_size, end_cursor=end_cursor, compress=compress, query=query,
+                                 accept_compress_type=accept_compress_type)
 
             yield res
             next_cursor = res.get_next_cursor()
@@ -1240,6 +1252,10 @@ class LogClient(object):
 
         :type query: string
         :param query: the SPL query, such as *| where a = 'xxx'
+        
+        :type accept_compress_type: str
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
 
         :return: LogResponse {"total_count": 30, "files": {'file_path_1': 10, "file_path_2": 20} })
 

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23}/aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.9.22'
+__version__ = '0.9.23'
 
 import sys
 OS_VERSION = str(sys.platform)

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.23/aliyun_log_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.22
+Version: 0.9.23
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun