aliyun-log-python-sdk 0.9.15__tar.gz → 0.9.16__tar.gz

{aliyun_log_python_sdk-0.9.15/aliyun_log_python_sdk.egg-info → aliyun_log_python_sdk-0.9.16}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: aliyun-log-python-sdk
-Version: 0.9.15
+Version: 0.9.16
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
@@ -23,11 +23,19 @@ Requires-Dist: elasticsearch
 Requires-Dist: jmespath
 Requires-Dist: dateparser
 Requires-Dist: protobuf<6.0.0,>=3.20.3
+Requires-Dist: lz4
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: lz4; extra == "test"
 Requires-Dist: virtualenv; extra == "test"
 Requires-Dist: zstandard; extra == "test"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 
 Python SDK for Alicloud Log Service

aliyun_log_python_sdk-0.9.16/aliyun/log/compress.py

@@ -0,0 +1,77 @@
+
+import zlib
+from enum import Enum
+from .logexception import LogException
+from .util import Util
+import six
+import lz4.block
+
+
+def lz_decompress(raw_size, data):
+    return lz4.block.decompress(data, uncompressed_size=raw_size)
+
+
+def lz_compresss(data):
+    return lz4.block.compress(data)[4:]
+
+
+class CompressType(Enum):
+    UNCOMPRESSED = 1
+    LZ4 = 2
+
+    @staticmethod
+    def default_compress_type():
+        return CompressType.LZ4
+
+    def __str__(self):
+        if self == CompressType.LZ4:
+            return 'lz4'
+        return ''
+
+
+class Compressor():
+    @staticmethod
+    def compress(data, compress_type):
+        # type: (bytes, CompressType) -> bytes
+        if compress_type == CompressType.LZ4:
+            return lz_compresss(data)
+
+        if compress_type == CompressType.UNCOMPRESSED:
+            return data
+
+        raise LogException('UnknownCompressType',
+                           'Unknown compress type: ' + compress_type)
+
+    @staticmethod
+    def decompress(data, raw_size, compress_type):
+        # type: (bytes, int, CompressType) -> bytes
+        if compress_type == CompressType.LZ4:
+            return lz_decompress(raw_size, data)
+
+        if compress_type == CompressType.UNCOMPRESSED:
+            return data
+
+        raise LogException('UnknownCompressType',
+                           'Unknown compress type: ' + compress_type)
+
+    @staticmethod
+    def decompress_response(header, response):
+        # type: (dict, bytes) -> bytes
+        compress_type_str = Util.h_v_td(
+            header, 'x-log-compresstype', '').lower()
+
+        if compress_type_str == '':
+            return response
+
+        raw_size = int(Util.h_v_td(header, 'x-log-bodyrawsize', 0))
+        if raw_size == 0:
+            return six.b('')
+
+        if compress_type_str == 'lz4':
+            return Compressor.decompress(response, raw_size, CompressType.LZ4)
+
+        if compress_type_str in ('gzip', 'deflate'):
+            return zlib.decompress(response)
+
+        raise LogException('UnknownCompressType', 'Unknown compress type: ' +
+                           compress_type_str, resp_header=header, resp_body=response)

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/aliyun/log/logclient.py

@@ -62,15 +62,13 @@ from .etl_config_response import *
 from .export_response import *
 from .common_response import *
 from .auth import *
+from .compress import CompressType, Compressor
 
 logger = logging.getLogger(__name__)
 
 if six.PY3:
     xrange = range
 
-lz4_available = Util.is_lz4_available()
-if lz4_available:
-    from .util import lz_decompress, lz_compresss
 
 CONNECTION_TIME_OUT = 120
 MAX_LIST_PAGING_SIZE = 500
@@ -384,13 +382,11 @@ class LogClient(object):
         raw_body_size = len(body)
         headers = {'x-log-bodyrawsize': str(raw_body_size), 'Content-Type': 'application/x-protobuf'}
 
-        if compress is None or compress:
-            if lz4_available:
-                headers['x-log-compresstype'] = 'lz4'
-                body = lz_compresss(body)
-            else:
-                headers['x-log-compresstype'] = 'deflate'
-                body = zlib.compress(body)
+        need_compress = compress is None or compress
+        if need_compress:
+            compress_type = CompressType.default_compress_type()
+            headers['x-log-compresstype'] = str(compress_type)
+            body = Compressor.compress(body, compress_type)
 
         params = {}
         resource = '/logstores/' + logstore + "/shards/lb"
@@ -445,16 +441,12 @@ class LogClient(object):
                                    len(body) / 1024.0 / 1024))
 
         headers = {'x-log-bodyrawsize': str(len(body)), 'Content-Type': 'application/x-protobuf'}
-        is_compress = request.get_compress()
-
-        compress_data = None
-        if is_compress:
-            if lz4_available:
-                headers['x-log-compresstype'] = 'lz4'
-                compress_data = lz_compresss(body)
-            else:
-                headers['x-log-compresstype'] = 'deflate'
-                compress_data = zlib.compress(body)
+        
+        need_compress = request.get_compress() is None or request.get_compress()
+        if need_compress:
+            compress_type = CompressType.default_compress_type()
+            headers['x-log-compresstype'] = str(compress_type)
+            body = Compressor.compress(body, compress_type)
 
         params = {}
         logstore = request.get_logstore()
@@ -465,11 +457,7 @@ class LogClient(object):
         else:
             resource = '/logstores/' + logstore + "/shards/lb"
 
-        if is_compress:
-            (resp, header) = self._send('POST', project, compress_data, resource, params, headers)
-        else:
-            (resp, header) = self._send('POST', project, body, resource, params, headers)
-
+        (resp, header) = self._send('POST', project, body, resource, params, headers)
         return PutLogsResponse(header, resp)
 
     def list_logstores(self, request):
@@ -654,12 +642,12 @@ class LogClient(object):
                 params['forward'] = forward
                 body_str = six.b(json.dumps(params))
                 headers["x-log-bodyrawsize"] = str(len(body_str))
-                accept_encoding = "lz4" if lz4_available else "deflate"
+                accept_encoding = str(CompressType.default_compress_type())
                 headers['Accept-Encoding'] = accept_encoding
 
                 (resp, header) = self._send("POST", project, body_str, resource, None, headers, respons_body_type=accept_encoding)
 
-                raw_data = Util.uncompress_response(header, resp)
+                raw_data = Compressor.decompress_response(header, resp)
                 exJson = self._loadJson(200, header, raw_data, requestId=Util.h_v_td(header, 'x-log-requestid', ''))
                 exJson = Util.convert_unicode_to_str(exJson)
                 ret = GetLogsResponse(exJson, header)
@@ -1128,11 +1116,10 @@ class LogClient(object):
         """
 
         headers = {}
-        if compress is None or compress:
-            if lz4_available:
-                headers['Accept-Encoding'] = 'lz4'
-            else:
-                headers['Accept-Encoding'] = 'gzip'
+
+        need_compress = compress is None or compress
+        if need_compress:
+            headers['Accept-Encoding'] = str(CompressType.default_compress_type())
         else:
             headers['Accept-Encoding'] = ''
 
@@ -1155,18 +1142,9 @@ class LogClient(object):
         raw_size = int(Util.h_v_t(header, 'x-log-bodyrawsize'))
         if raw_size <= 0:
             return PullLogResponse(None, header)
-        compress_type = Util.h_v_td(header, 'x-log-compresstype', '').lower()
-        if compress_type == 'lz4':
-            if lz4_available:
-                raw_data = lz_decompress(raw_size, resp)
-                return PullLogResponse(raw_data, header)
-            else:
-                raise LogException("ClientHasNoLz4", "There's no Lz4 lib available to decompress the response", resp_header=header, resp_body=resp)
-        elif compress_type in ('gzip', 'deflate'):
-            raw_data = zlib.decompress(resp)
-            return PullLogResponse(raw_data, header)
-        else:
-            return PullLogResponse(resp, header)
+
+        raw_data = Compressor.decompress_response(header, resp)
+        return PullLogResponse(raw_data, header)
 
     def pull_log(self, project_name, logstore_name, shard_id, from_time, to_time, batch_size=None, compress=None, query=None):
         """ batch pull log data from log service using time-range

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/aliyun/log/putlogsrequest.py

@@ -29,7 +29,7 @@ class PutLogsRequest(LogRequest):
     :param hashKey: put data with set hash, the data will be send to shard whose range contains the hashKey
 
     :type compress: bool
-    :param compress: if need to compress the logs
+    :param compress: if need to compress the logs, default is True
 
     :type logtags: list
     :param logtags: list of key:value tag pair , [(tag_key_1,tag_value_1) , (tag_key_2,tag_value_2)]

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/aliyun/log/util.py

@@ -17,27 +17,7 @@ from .logexception import LogException
 import re
 import logging
 import json
-import struct
-import zlib
 
-try:
-    import lz4
-    if hasattr(lz4, 'loads') and hasattr(lz4, 'dumps'):
-        def lz_decompress(raw_size, data):
-            return lz4.loads(struct.pack('<I', raw_size) + data)
-
-        def lz_compresss(data):
-            return lz4.dumps(data)[4:]
-    else:
-        import lz4.block
-        def lz_decompress(raw_size, data):
-            return lz4.block.decompress(data, uncompressed_size=raw_size)
-
-        def lz_compresss(data):
-            return lz4.block.compress(data)[4:]
-    lz4_available = True
-except ImportError:
-    lz4_available = False
 
 logger = logging.getLogger(__name__)
 
@@ -91,11 +71,6 @@ class Util(object):
             if s:
                 s.close()
 
-    @staticmethod
-    def compress_data(data):
-        import zlib
-        return zlib.compress(data, 6)
-
     @staticmethod
     def cal_md5(content):
         return hashlib.md5(content).hexdigest().upper()
@@ -164,9 +139,6 @@ class Util(object):
 
         return data
 
-    @staticmethod
-    def is_lz4_available():
-        return lz4_available
 
     @staticmethod
     def h_v_t(header, key):
@@ -208,23 +180,7 @@ class Util(object):
             else return v
         """
         return v if v is not None else default
-
-    @staticmethod
-    def uncompress_response(header, response):
-        """
-        Supported compress type [lz4, gzip, deflate]
-        """
-        compress_type = Util.h_v_td(header, 'x-log-compresstype', '').lower()
-        if compress_type == 'lz4':
-            raw_size = int(Util.h_v_t(header, 'x-log-bodyrawsize'))
-            if Util.is_lz4_available():
-                return lz_decompress(raw_size, response)
-            else:
-                raise LogException("ClientHasNoLz4", "There's no Lz4 lib available to decompress the response", resp_header=header, resp_body=response)
-        elif compress_type in ('gzip', 'deflate'):
-            return zlib.decompress(response)
-        else:
-            raise LogException('UnknownCompressType', 'Unknown compress type: ' + compress_type, resp_header=header, resp_body=response)
+    
 
 ZERO = timedelta(0)
 

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.9.15'
+__version__ = '0.9.16'
 
 
 import sys

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16/aliyun_log_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: aliyun-log-python-sdk
-Version: 0.9.15
+Version: 0.9.16
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
@@ -23,11 +23,19 @@ Requires-Dist: elasticsearch
 Requires-Dist: jmespath
 Requires-Dist: dateparser
 Requires-Dist: protobuf<6.0.0,>=3.20.3
+Requires-Dist: lz4
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: lz4; extra == "test"
 Requires-Dist: virtualenv; extra == "test"
 Requires-Dist: zstandard; extra == "test"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 
 Python SDK for Alicloud Log Service

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/aliyun_log_python_sdk.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ aliyun/log/acl_config.py
 aliyun/log/acl_response.py
 aliyun/log/auth.py
 aliyun/log/common_response.py
+aliyun/log/compress.py
 aliyun/log/consumer_group_request.py
 aliyun/log/consumer_group_response.py
 aliyun/log/credentials.py

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/aliyun_log_python_sdk.egg-info/requires.txt

@@ -5,6 +5,7 @@ elasticsearch
 jmespath
 dateparser
 protobuf<6.0.0,>=3.20.3
+lz4
 
 [test]
 pytest

{aliyun_log_python_sdk-0.9.15 → aliyun_log_python_sdk-0.9.16}/setup.py

@@ -33,6 +33,7 @@ requirements_py3 = [
     'jmespath',
     'dateparser',
     'protobuf>=3.20.3,<6.0.0',
+    'lz4',
 ]
 requirements_py2 = [
     'six==1.14.0',
@@ -45,7 +46,7 @@ requirements_py2 = [
     'protobuf>3.4.0,<=3.17.3', # 3.18.0 is the last version support py2, but yanked
     'regex==2021.3.17',
     'tzlocal==2.0.0',
-    'lz4a==0.7.0',
+    'lz4==2.2.1',
 ]
 
 test_requirements = [