aliyun-log-python-sdk 0.9.22__tar.gz → 0.9.24__tar.gz

{aliyun_log_python_sdk-0.9.22/aliyun_log_python_sdk.egg-info → aliyun_log_python_sdk-0.9.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.22
+Version: 0.9.24
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/__init__.py

@@ -38,3 +38,6 @@ from .rebuild_index_response import *
 
 # logging handler
 from .logger_hanlder import SimpleLogHandler, QueuedLogHandler, LogFields, UwsgiQueuedLogHandler
+from .metering_mode_response import GetLogStoreMeteringModeResponse, \
+    GetMetricStoreMeteringModeResponse, \
+    UpdateLogStoreMeteringModeResponse, UpdateMetricStoreMeteringModeResponse

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/compress.py

@@ -20,10 +20,19 @@ def lz_compresss(data):
 
 def zstd_compress(data):
     if not zstd_available:
-        raise LogException('UnsupportedompressType',
-                           'Unsupported compress type zstd, zstd not installed')
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type zstd, library zstd not installed')
     return zstd.compress(data, 1)
 
+def zstd_decompress(raw_size, data):
+    if not zstd_available:
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type zstd, library zstd not installed')
+    result = zstd.decompress(data)
+    if len(result) != raw_size:
+        raise LogException('DecompressError', 'uncompressed size ' + str(len(result)) + ' does not match x-log-bodyrawsize ' + str(raw_size))
+    return result
+
 class CompressType(Enum):
     UNCOMPRESSED = 1
     LZ4 = 2
@@ -62,8 +71,8 @@ class Compressor():
         if compress_type == CompressType.UNCOMPRESSED:
             return data
 
-        raise LogException('UnknownCompressType',
-                           'Unknown compress type: ' + compress_type)
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type: ' + compress_type)
 
     @staticmethod
     def decompress(data, raw_size, compress_type):
@@ -73,9 +82,12 @@ class Compressor():
 
         if compress_type == CompressType.UNCOMPRESSED:
             return data
+        
+        if compress_type == CompressType.ZSTD:
+            return zstd_decompress(raw_size, data)
 
-        raise LogException('UnknownCompressType',
-                           'Unknown compress type: ' + compress_type)
+        raise LogException('UnsupportedCompressType',
+                           'Unsupported compress type: ' + compress_type)
 
     @staticmethod
     def decompress_response(header, response):
@@ -93,8 +105,11 @@ class Compressor():
         if compress_type_str == 'lz4':
             return Compressor.decompress(response, raw_size, CompressType.LZ4)
 
+        if compress_type_str == 'zstd':
+            return Compressor.decompress(response, raw_size, CompressType.ZSTD)
+
         if compress_type_str in ('gzip', 'deflate'):
             return zlib.decompress(response)
 
-        raise LogException('UnknownCompressType', 'Unknown compress type: ' +
+        raise LogException('UnsupportedCompressType', 'Unsupported compress type: ' +
                            compress_type_str, resp_header=header, resp_body=response)

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/consumer/config.py

@@ -25,7 +25,8 @@ class LogHubConfig(object):
                  cursor_position=None, heartbeat_interval=None, data_fetch_interval=None, in_order=False,
                  cursor_start_time=None, security_token=None, max_fetch_log_group_size=None, worker_pool_size=None, shared_executor=None,
                  cursor_end_time=None, credentials_refresher=None,
-                 auth_version=AUTH_VERSION_1, region='', query=None):
+                 auth_version=AUTH_VERSION_1, region='', query=None,
+                 accept_compress_type=None):
         """
 
         :param endpoint:
@@ -48,6 +49,10 @@ class LogHubConfig(object):
         :param auth_version: only support AUTH_VERSION_1 and AUTH_VERSION_4
         :param region: region of project
         :param query: the SPL query, such as *| where a = 'xxx'
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
+
         """
         self.endpoint = endpoint
         self.accessKeyId = access_key_id
@@ -71,3 +76,4 @@ class LogHubConfig(object):
         self.auth_version = auth_version
         self.region = region
         self.query = query
+        self.accept_compress_type = accept_compress_type

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/consumer/consumer_client.py

@@ -22,18 +22,60 @@ class ConsumerClientLoggerAdapter(logging.LoggerAdapter):
 class ConsumerClient(object):
     def __init__(self, endpoint, access_key_id, access_key, project,
                  logstore, consumer_group, consumer, security_token=None, credentials_refresher=None,
-                 auth_version=AUTH_VERSION_1, region=''):
+                 auth_version=AUTH_VERSION_1, region='',
+                 accept_compress_type=None):
+        '''
+        :type endpoint: string
+        :param endpoint: the endpoint of sls project
 
+        :type access_key_id: string
+        :param access_key_id: your access key id
+
+        :type access_key: string
+        :param access_key: your access key secret
+
+        :type project: string
+        :param project: sls project
+
+        :type logstore: string
+        :param logstore: sls logstore to consume
+
+        :type consumer_group: string
+        :param consumer_group: consumer group name, the consumer group will be created when needed
+
+        :type consumer: string
+        :param consumer: consumer id, the id must be unique in the same consumer group
+
+        :type security_token: string
+        :param security_token: security token
+
+        :type credentials_refresher: CredentialsProvider
+        :param credentials_refresher: provides temporarily accessKeyId & accessKey dynamically
+
+        :type auth_version: string
+        :param auth_version: the auth version for signature, eg. AUTH_VERSION_1, AUTH_VERSION_4
+
+        :type region: string
+        :param region: the region of your project, required when auth_version is AUTH_VERSION_4
+
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
+
+        '''
         from .. import LogClient
 
-        self.mclient = LogClient(endpoint, access_key_id, access_key, security_token, auth_version=auth_version, region=region)
-        self.mclient.set_user_agent('%s-consumergroup-%s-%s' % (USER_AGENT, consumer_group, consumer))
+        self.mclient = LogClient(endpoint, access_key_id, access_key,
+                                 security_token, auth_version=auth_version, region=region)
+        self.mclient.set_user_agent(
+            '%s-consumergroup-%s-%s' % (USER_AGENT, consumer_group, consumer))
         if credentials_refresher is not None:
             self.mclient.set_credentials_auto_refresher(credentials_refresher)
         self.mproject = project
         self.mlogstore = logstore
         self.mconsumer_group = consumer_group
         self.mconsumer = consumer
+        self.accept_compress_type = accept_compress_type
         self.logger = ConsumerClientLoggerAdapter(
             logging.getLogger(__name__), {"consumer_client": self})
 
@@ -72,8 +114,8 @@ class ConsumerClient(object):
         except LogException as _:
             return None, True
 
-
     # fallback, for backward compatibility
+
     def _ensure_consumer_group_created_legacy(self, timeout, in_order):
         try:
             self.mclient.create_consumer_group(self.mproject, self.mlogstore, self.mconsumer_group,
@@ -138,4 +180,4 @@ class ConsumerClient(object):
         return self.mclient.get_end_cursor(self.mproject, self.mlogstore, shard_id).get_cursor()
 
     def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None):
-        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor,query=query)
+        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor, query=query, accept_compress_type=self.accept_compress_type)

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/consumer/worker.py

@@ -36,7 +36,8 @@ class ConsumerWorker(Thread):
                            consumer_option.project, consumer_option.logstore, consumer_option.consumer_group_name,
                            consumer_option.consumer_name, consumer_option.securityToken,
                            credentials_refresher=consumer_option.credentials_refresher,
-                           auth_version=consumer_option.auth_version, region=consumer_option.region)
+                           auth_version=consumer_option.auth_version, region=consumer_option.region,
+                           accept_compress_type=consumer_option.accept_compress_type)
         self.shut_down_flag = False
         self.logger = ConsumerWorkerLoggerAdapter(
             logging.getLogger(__name__), {"consumer_worker": self})

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/logclient.py

@@ -63,6 +63,9 @@ from .export_response import *
 from .common_response import *
 from .auth import *
 from .compress import CompressType, Compressor
+from .metering_mode_response import GetLogStoreMeteringModeResponse, \
+    GetMetricStoreMeteringModeResponse, UpdateLogStoreMeteringModeResponse, \
+        UpdateMetricStoreMeteringModeResponse
 
 logger = logging.getLogger(__name__)
 
@@ -1094,7 +1097,7 @@ class LogClient(object):
         """
         return self.get_cursor(project_name, logstore_name, shard_id, "end")
 
-    def pull_logs(self, project_name, logstore_name, shard_id, cursor, count=None, end_cursor=None, compress=None, query=None):
+    def pull_logs(self, project_name, logstore_name, shard_id, cursor, count=None, end_cursor=None, compress=None, query=None, accept_compress_type=None):
         """ batch pull log data from log service
         Unsuccessful operation will cause an LogException.
 
@@ -1121,6 +1124,10 @@ class LogClient(object):
 
         :type query: string
         :param query: the SPL query, such as *| where a = 'xxx'
+        
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
 
         :return: PullLogResponse
 
@@ -1131,7 +1138,10 @@ class LogClient(object):
 
         need_compress = compress is None or compress
         if need_compress:
-            headers['Accept-Encoding'] = str(CompressType.default_compress_type())
+            if accept_compress_type is None or accept_compress_type == '':
+                headers['Accept-Encoding'] = str(CompressType.default_compress_type())
+            else:
+                headers['Accept-Encoding'] = accept_compress_type
         else:
             headers['Accept-Encoding'] = ''
 
@@ -1158,7 +1168,7 @@ class LogClient(object):
         raw_data = Compressor.decompress_response(header, resp)
         return PullLogResponse(raw_data, header)
 
-    def pull_log(self, project_name, logstore_name, shard_id, from_time, to_time, batch_size=None, compress=None, query=None):
+    def pull_log(self, project_name, logstore_name, shard_id, from_time, to_time, batch_size=None, compress=None, query=None, accept_compress_type=None):
         """ batch pull log data from log service using time-range
         Unsuccessful operation will cause an LogException. the time parameter means the time when server receives the logs
 
@@ -1185,6 +1195,10 @@ class LogClient(object):
 
         :type query: string
         :param query: the SPL query, such as *| where a = 'xxx'
+        
+        :type accept_compress_type: string
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
 
         :return: PullLogResponse
 
@@ -1195,7 +1209,8 @@ class LogClient(object):
 
         while True:
             res = self.pull_logs(project_name, logstore_name, shard_id, begin_cursor,
-                                 count=batch_size, end_cursor=end_cursor, compress=compress, query=query)
+                                 count=batch_size, end_cursor=end_cursor, compress=compress, query=query,
+                                 accept_compress_type=accept_compress_type)
 
             yield res
             next_cursor = res.get_next_cursor()
@@ -1240,6 +1255,10 @@ class LogClient(object):
 
         :type query: string
         :param query: the SPL query, such as *| where a = 'xxx'
+        
+        :type accept_compress_type: str
+        :param accept_compress_type: The compression type used for logs retrieved from sls.
+        Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
 
         :return: LogResponse {"total_count": 30, "files": {'file_path_1': 10, "file_path_2": 20} })
 
@@ -1569,6 +1588,104 @@ class LogClient(object):
         (resp, header) = self._send("GET", project_name, None, resource, params, headers)
         return ListLogStoreResponse(resp, header)
 
+    def get_logstore_metering_mode(self, project_name, logstore_name):
+        """ Get the metering mode of the logstore
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type logstore_name: string
+        :param logstore_name: the logstore name
+
+        :return: GetLogStoreMeteringModeResponse
+
+        :raise: LogException
+        """
+        params = {}
+        resource = "/logstores/" + logstore_name + "/meteringmode"
+        headers = {}
+
+        (resp, header) = self._send("GET", project_name, None, resource, params, headers)
+        return GetLogStoreMeteringModeResponse(resp, header)
+    
+    def update_logstore_metering_mode(self, project_name, logstore_name, metering_mode):
+        """ Update the metering mode of the logstore
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type logstore_name: string
+        :param logstore_name: the logstore name
+        
+        :type metering_mode: string
+        :param metering_mode: the metering mode, ChargeByDataIngest or ChargeByFunction
+
+        :return: UpdateLogStoreMeteringModeResponse
+
+        :raise: LogException
+        """
+        params = {}
+        resource = "/logstores/" + logstore_name + "/meteringmode"
+        headers = {"x-log-bodyrawsize": '0', "Content-Type": "application/json"}
+        body = {
+            "meteringMode": metering_mode
+        }
+        body_str = six.b(json.dumps(body))
+
+        (resp, header) = self._send("PUT", project_name, body_str, resource, params, headers)
+        return UpdateLogStoreMeteringModeResponse(header, resp)
+    
+    def get_metric_store_metering_mode(self, project_name, metric_store_name):
+        """ Get the metering mode of the metric store
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type metric_store_name: string
+        :param metric_store_name: the metric store name
+
+        :return: GetMetricStoreMeteringModeResponse
+
+        :raise: LogException
+        """
+        params = {}
+        resource = "/metricstores/" + metric_store_name + "/meteringmode"
+        headers = {}
+
+        (resp, header) = self._send("GET", project_name, None, resource, params, headers)
+        return GetMetricStoreMeteringModeResponse(resp, header)
+    
+    def update_metric_store_metering_mode(self, project_name, metric_store_name, metering_mode):
+        """ Update the metering mode of the metric store
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type metric_store_name: string
+        :param metric_store_name: the metric store name
+        
+        :type metering_mode: string
+        :param metering_mode: the metering mode, ChargeByDataIngest or ChargeByFunction
+
+        :return: UpdateMetricStoreMeteringModeResponse
+
+        :raise: LogException
+        """
+        params = {}
+        resource = "/metricstores/" + metric_store_name + "/meteringmode"
+        headers = {"x-log-bodyrawsize": '0', "Content-Type": "application/json"}
+        body = {
+            "meteringMode": metering_mode
+        }
+        body_str = six.b(json.dumps(body))
+
+        (resp, header) = self._send("PUT", project_name, body_str, resource, params, headers)
+        return UpdateMetricStoreMeteringModeResponse(header, resp)
+
     def create_external_store(self, project_name, config):
         """ create log store
         Unsuccessful operation will cause an LogException.

aliyun_log_python_sdk-0.9.24/aliyun/log/metering_mode_response.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .util import Util
+from .logresponse import LogResponse
+
+
+class GetLogStoreMeteringModeResponse(LogResponse):
+    """ The response of the get_logstore_metering_mode API from log.
+
+    :type header: dict
+    :param header: GetLogStoreMeteringModeResponse HTTP response header
+
+    :type resp: dict
+    :param resp: the HTTP response body
+    """
+
+    def __init__(self, resp, header):
+        LogResponse.__init__(self, header, resp)
+        self.metering_mode = Util.convert_unicode_to_str(resp["meteringMode"])
+
+    def get_metering_mode(self):
+        """
+
+        :return: string, the metering mode of the logstore
+        """
+        return self.metering_mode
+
+    def log_print(self):
+        print('GetLogStoreMeteringModeResponse')
+        print('headers:', self.get_all_headers())
+        print('meteringMode:', self.metering_mode)
+
+class GetMetricStoreMeteringModeResponse(LogResponse):
+    """ The response of the get_metric_store_metering_mode API from log.
+
+    :type header: dict
+    :param header: GetMetricStoreMeteringModeResponse HTTP response header
+
+    :type resp: dict
+    :param resp: the HTTP response body
+    """
+
+    def __init__(self, resp, header):
+        LogResponse.__init__(self, header, resp)
+        self.metering_mode = Util.convert_unicode_to_str(resp["meteringMode"])
+
+    def get_metering_mode(self):
+        """
+        :return: string, the metering mode of the metric store
+        """
+        return self.metering_mode
+
+    def log_print(self):
+        print('GetMetricStoreMeteringModeResponse')
+        print('headers:', self.get_all_headers())
+        print('meteringMode:', self.metering_mode)
+        
+class UpdateLogStoreMeteringModeResponse(LogResponse):
+    """ The response of the update_logstore_metering_mode API from log.
+
+    :type header: dict
+    :param header: UpdateLogStoreMeteringModeResponse HTTP response header
+    """
+
+    def __init__(self, header, resp=''):
+        LogResponse.__init__(self, header, resp)
+
+    def log_print(self):
+        print('UpdateLogStoreMeteringModeResponse:')
+        print('headers:', self.get_all_headers())
+        
+class UpdateMetricStoreMeteringModeResponse(LogResponse):
+    """ The response of the update_metric_store_metering_mode API from log.
+
+    :type header: dict
+    :param header: UpdateMetricStoreMeteringModeResponse HTTP response header
+    """
+
+    def __init__(self, header, resp=''):
+        LogResponse.__init__(self, header, resp)
+
+    def log_print(self):
+        print('UpdateMetricStoreMeteringModeResponse:')
+        print('headers:', self.get_all_headers())

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.9.22'
+__version__ = '0.9.24'
 
 import sys
 OS_VERSION = str(sys.platform)

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24/aliyun_log_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.22
+Version: 0.9.24
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun

{aliyun_log_python_sdk-0.9.22 → aliyun_log_python_sdk-0.9.24}/aliyun_log_python_sdk.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ aliyun/log/logtail_config_detail.py
 aliyun/log/logtail_config_response.py
 aliyun/log/machine_group_detail.py
 aliyun/log/machinegroup_response.py
+aliyun/log/metering_mode_response.py
 aliyun/log/odps_sink.py
 aliyun/log/oss_sink.py
 aliyun/log/pluralize.py