aliyun-log-python-sdk 0.9.32__tar.gz → 0.9.34__tar.gz

{aliyun_log_python_sdk-0.9.32/aliyun_log_python_sdk.egg-info → aliyun_log_python_sdk-0.9.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.32
+Version: 0.9.34
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/consumer/__init__.py

@@ -1,4 +1,4 @@
 
-from .tasks import ConsumerProcessorBase, ConsumerProcessorAdaptor
+from .tasks import ConsumerProcessorBase, ConsumerProcessorAdaptor, ConsumerJsonProcessorBase
 from .config import LogHubConfig, CursorPosition
 from .worker import ConsumerWorker

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/consumer/consumer_client.py

@@ -7,6 +7,7 @@ from .exceptions import ClientWorkerException
 from ..logexception import LogException
 from ..version import USER_AGENT
 from ..auth import *
+from ..pulllog_response import PullLogResponse
 
 
 class ConsumerClientLoggerAdapter(logging.LoggerAdapter):
@@ -180,4 +181,5 @@ class ConsumerClient(object):
         return self.mclient.get_end_cursor(self.mproject, self.mlogstore, shard_id).get_cursor()
 
     def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None, processor=None):
+        # type: (int, str, str, int, str, object) -> PullLogResponse
         return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor, query=query, accept_compress_type=self.accept_compress_type, processor=processor)

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/consumer/fetched_log_group.py

@@ -3,18 +3,19 @@
 
 class FetchedLogGroup(object):
 
-    def __init__(self, shard_id, log_group_list, end_cursor):
+    def __init__(self, shard_id, data, end_cursor, log_group_count):
         self._shard_id = shard_id
-        self._fetched_log_group_list = log_group_list
+        self._data = data
         self._end_cursor = end_cursor
+        self._log_group_count = log_group_count
 
     @property
     def shard_id(self):
         return self._shard_id
 
     @property
-    def fetched_log_group_list(self):
-        return self._fetched_log_group_list
+    def data(self):
+        return self._data
 
     @property
     def end_cursor(self):
@@ -22,4 +23,4 @@ class FetchedLogGroup(object):
 
     @property
     def log_group_size(self):
-        return len(self._fetched_log_group_list.LogGroups)
+        return self._log_group_count

aliyun_log_python_sdk-0.9.34/aliyun/log/consumer/preprocessor.py

@@ -0,0 +1,6 @@
+def to_log_group_list_pb(pull_log_response):
+    return pull_log_response.get_loggroup_list()
+
+
+def to_flattern_json_list(pull_log_response):
+    return pull_log_response.get_flatten_logs_json()

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/consumer/shard_worker.py

@@ -66,6 +66,7 @@ class ShardConsumerWorker(object):
         self.save_last_checkpoint = False
         self.query = query
         self.consume_processor = consume_processor
+        self.preprocessor = self.processor._preprocessor
         self.logger = ShardConsumerWorkerLoggerAdapter(
             logging.getLogger(__name__), {"shard_consumer_worker": self})
 
@@ -98,10 +99,13 @@ class ShardConsumerWorker(object):
 
                 self.last_success_fetch_time = time.time()
 
-                self.last_fetch_log_group = FetchedLogGroup(self.shard_id, task_result.get_fetched_log_group_list(),
-                                                            task_result.get_cursor())
+                self.last_fetch_log_group = FetchedLogGroup(self.shard_id,
+                                                            task_result.get_data(),
+                                                            task_result.get_cursor(),
+                                                            task_result.get_log_group_count())
+
                 self.next_fetch_cursor = task_result.get_cursor()
-                self.last_fetch_count = self.last_fetch_log_group.log_group_size
+                self.last_fetch_count = task_result.get_log_group_count()
                 self.last_fetch_size = task_result.get_raw_size()
                 self.rawLogGroupCountBeforeQuery = task_result.get_raw_log_group_count_before_query()
                 self.rawSizeBeforeQuery = task_result.get_raw_size_before_query()
@@ -114,7 +118,6 @@ class ShardConsumerWorker(object):
                         self.checkpoint_tracker.flush_check_point()
                         self.save_last_checkpoint = True
 
-
             self._sample_log_error(task_result)
 
             # no task or task is done, create new task
@@ -136,12 +139,17 @@ class ShardConsumerWorker(object):
                     is_generate_fetch_task = (time.time() - self.last_fetch_time) > 0.05
                 if is_generate_fetch_task:
                     self.last_fetch_time = time.time()
-                    self.fetch_data_future = \
-                        self.executor.submit(consumer_fetch_task,
-                                             self.log_client, self.shard_id, self.next_fetch_cursor,
-                                             max_fetch_log_group_size=self.max_fetch_log_group_size,
-                                             end_cursor=self.fetch_end_cursor, query=self.query,
-                                             consume_processor=self.consume_processor)
+                    self.fetch_data_future = self.executor.submit(
+                        consumer_fetch_task,
+                        self.log_client,
+                        self.preprocessor,
+                        self.shard_id,
+                        self.next_fetch_cursor,
+                        max_fetch_log_group_size=self.max_fetch_log_group_size,
+                        end_cursor=self.fetch_end_cursor,
+                        query=self.query,
+                        consume_processor=self.consume_processor,
+                    )
                 else:
                     self.fetch_data_future = None
             else:
@@ -213,13 +221,12 @@ class ShardConsumerWorker(object):
                 self.checkpoint_tracker.set_cursor(self.last_fetch_log_group.end_cursor)
                 self.current_task_exist = True
 
-                # must deep copy cause some revision will happen
-                last_fetch_log_group = copy.deepcopy(self.last_fetch_log_group)
+                log_group = self.last_fetch_log_group
                 self.last_fetch_log_group = None
 
                 if self.last_fetch_count > 0:
                     self.task_future = self.executor.submit(consumer_process_task, self.processor,
-                                                            last_fetch_log_group.fetched_log_group_list,
+                                                            log_group.data,
                                                             self.checkpoint_tracker)
 
         elif self.consumer_status == ConsumerStatus.SHUTTING_DOWN:

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/consumer/tasks.py

@@ -5,6 +5,9 @@ import logging
 
 from .config import CursorPosition
 from ..logexception import LogException
+from ..pulllog_response import PullLogResponse
+from .consumer_client import ConsumerClient
+from .preprocessor import to_log_group_list_pb, to_flattern_json_list
 import time
 import six
 import sys
@@ -17,6 +20,7 @@ class ConsumerProcessorBase(object):
         self.shard_id = -1
         self.last_check_time = 0
         self.checkpoint_timeout = 3
+        self._preprocessor = to_log_group_list_pb
 
     def save_checkpoint(self, tracker, force=False):
         current_time = time.time()
@@ -53,6 +57,20 @@ class ConsumerProcessorBase(object):
         self.save_checkpoint(check_point_tracker, force=True)
 
 
+class ConsumerJsonProcessorBase(ConsumerProcessorBase):
+
+    def __init__(self):
+        super(ConsumerJsonProcessorBase, self).__init__()
+        self._preprocessor = to_flattern_json_list
+
+    @abc.abstractmethod
+    def process(self, flattern_json_list, check_point_tracker):
+        """
+        @flattern_json_list a list of dict, each dict is a log, eg: [{"__time__": 1761721979, "key1": "value1", "hello": "world", "__tag__:tagkey1": "tagvalue1" }]
+        """
+        raise NotImplementedError("not create method process")
+
+
 class ConsumerProcessorAdaptor(ConsumerProcessorBase):
     def __init__(self, func):
         super(ConsumerProcessorAdaptor, self).__init__()
@@ -112,28 +130,32 @@ class InitTaskResult(TaskResult):
 
 
 class FetchTaskResult(TaskResult):
-    def __init__(self, fetched_log_group_list, cursor, raw_size , raw_size_before_query, raw_log_group_count_before_query):
+    def __init__(self, data, log_group_count, cursor, raw_size , raw_size_before_query, raw_log_group_count_before_query):
         super(FetchTaskResult, self).__init__(None)
-        self.fetched_log_group_list = fetched_log_group_list
-        self.cursor = cursor
-        self.raw_size = raw_size
-        self.raw_size_before_query = raw_size_before_query
-        self.raw_log_group_count_before_query = raw_log_group_count_before_query
+        self._data = data
+        self._log_group_count = log_group_count
+        self._cursor = cursor
+        self._raw_size = raw_size
+        self._raw_size_before_query = raw_size_before_query
+        self._raw_log_group_count_before_query = raw_log_group_count_before_query
 
-    def get_fetched_log_group_list(self):
-        return self.fetched_log_group_list
+    def get_data(self):
+        return self._data
 
     def get_cursor(self):
-        return self.cursor
+        return self._cursor
+
+    def get_log_group_count(self):
+        return self._log_group_count
 
     def get_raw_size(self):
-        return self.raw_size
+        return self._raw_size
 
     def get_raw_size_before_query(self):
-        return self.raw_size_before_query
+        return self._raw_size_before_query
 
     def get_raw_log_group_count_before_query(self):
-        return self.raw_log_group_count_before_query
+        return self._raw_log_group_count_before_query
 
 
 def consumer_process_task(processor, log_groups, check_point_tracker):
@@ -186,16 +208,18 @@ def consumer_initialize_task(processor, consumer_client, shard_id, cursor_positi
     except Exception as e:
         return TaskResult(e)
 
-
-def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None, consume_processor=None):
+def consumer_fetch_task(loghub_client_adapter, preprocessor, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None, consume_processor=None):
+    # type: (ConsumerClient, object, int, str, int, str, str, str) -> FetchTaskResult
     exception = None
 
     for retry_times in range(3):
         try:
+            # type: PullLogResponse
             response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query, processor=consume_processor)
-            fetch_log_group_list = response.get_loggroup_list()
+            data = preprocessor(response)
             next_cursor = response.get_next_cursor()
             raw_size = response.get_raw_size()
+            log_group_count = response.get_loggroup_count()
             raw_size_before_query = 0
             raw_log_group_count_before_query = 0
             if query or consume_processor:
@@ -203,11 +227,11 @@ def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_g
                 raw_log_group_count_before_query = max(response.get_raw_log_group_count_before_query(), 0)
             logger.debug("shard id = %s cursor = %s next cursor = %s size: %s",
                          shard_id, cursor, next_cursor,
-                         response.get_log_count())
+                         log_group_count)
             if not next_cursor:
-                return FetchTaskResult(fetch_log_group_list, cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
+                return FetchTaskResult(data, log_group_count, cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
             else:
-                return FetchTaskResult(fetch_log_group_list, next_cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
+                return FetchTaskResult(data, log_group_count, next_cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
         except LogException as e:
             exception = e
             if exception.get_resp_status() == 403:

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/machine_group_detail.py

@@ -71,13 +71,19 @@ class MachineStatus(object):
     :param heartbeat_time: last updated heartbeat_time
     """
 
-    def __init__(self, ip, machine_unique_id, user_defined_id, heartbeat_time):
+    def __init__(self, ip, machine_unique_id, user_defined_id, heartbeat_time, binary=None):
         self.ip = ip
         self.machine_unique_id = machine_unique_id
         self.user_defined_id = user_defined_id
         self.heartbeat_time = heartbeat_time
+        self.binary = binary
 
     def log_print(self):
-        print('MachineStatus: machine_unique_id={}, user_defined_id={}, heartbeat_time={}'.format(self.machine_unique_id,
-                                                                                                  self.user_defined_id,
-                                                                                                  self.heartbeat_time))
+        print(
+            "MachineStatus: machine_unique_id={}, user_defined_id={}, heartbeat_time={}, binary={}".format(
+                self.machine_unique_id,
+                self.user_defined_id,
+                self.heartbeat_time,
+                self.binary,
+            )
+        )

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/machinegroup_response.py

@@ -181,12 +181,17 @@ class ListMachinesResponse(LogResponse):
 
     def __init__(self, resp, header):
         LogResponse.__init__(self, header, resp)
-        self.count = resp['count']
-        self.total = resp['total']
+        self.count = resp["count"]
+        self.total = resp["total"]
         self.machines = []
-        for machine_status in resp['machines']:
-            machine = MachineStatus(machine_status['ip'], machine_status['machine-uniqueid'],
-                                    machine_status['userdefined-id'], machine_status['lastHeartbeatTime'])
+        for machine_status in resp["machines"]:
+            machine = MachineStatus(
+                machine_status["ip"],
+                machine_status.get("machine-uniqueid", None),
+                machine_status["userdefined-id"],
+                machine_status["lastHeartbeatTime"],
+                machine_status.get("binary", None),
+            )
             self.machines.append(machine)
 
     def get_machine_count(self):

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/pulllog_response.py

@@ -28,18 +28,28 @@ class PullLogResponse(LogResponse):
     def __init__(self, resp, header):
         LogResponse.__init__(self, header, resp)
         self._is_bytes_type = None
-        self.log_count = int(Util.h_v_t(header, "x-log-count"))
+        self._log_group_count = int(Util.h_v_t(header, "x-log-count"))
         self.raw_size = int(Util.h_v_t(header, 'x-log-bodyrawsize'))
         self.next_cursor = Util.convert_unicode_to_str(Util.h_v_t(header, "x-log-cursor"))
         self.raw_log_group_count_before_query = int(Util.h_v_td(self.headers, 'x-log-rawdatacount', '-1'))
         self.raw_size_before_query = int(Util.h_v_td(self.headers, 'x-log-rawdatasize', '-1'))
-        self.loggroup_list = LogGroupList()
-        if resp is not None:
-            self._parse_loggroup_list(resp)
+
+        self._loggroup_list = None
+        self._raw_uncompressed_body = resp
         self.loggroup_list_json = None
         self.flatten_logs_json = None
         self._body = None
 
+
+    @property
+    def loggroup_list(self):
+        if self._loggroup_list is None :
+            log_group_list, is_bytes_type = self._parse_loggroup_list(self._raw_uncompressed_body)
+            self._loggroup_list = log_group_list
+            self._is_bytes_type = is_bytes_type
+            self._raw_uncompressed_body = None
+        return self._loggroup_list
+
     def get_body(self):
         if self._body is None:
             self._body = {"next_cursor": self.next_cursor,
@@ -58,11 +68,15 @@ class PullLogResponse(LogResponse):
     def get_next_cursor(self):
         return self.next_cursor
 
+    @property
+    def log_count(self):
+        return self.get_log_count()
+
     def get_log_count(self):
         return len(self.get_flatten_logs_json())
 
     def get_loggroup_count(self):
-        return len(self.loggroup_list.LogGroups)
+        return self._log_group_count
 
     def get_loggroup_json_list(self):
         if self.loggroup_list_json is None:
@@ -89,21 +103,24 @@ class PullLogResponse(LogResponse):
     def log_print(self):
         print('PullLogResponse')
         print('next_cursor', self.next_cursor)
-        print('log_count', self.log_count)
+        print('log_group_count', self.get_loggroup_count())
         print('headers:', self.get_all_headers())
         print('detail:', self.get_loggroup_json_list())
 
     def _parse_loggroup_list(self, data):
+        if data is None:
+            return LogGroupList(), False 
+
         try:
-            self.loggroup_list.ParseFromString(data)
+            p = LogGroupList()
+            p.ParseFromString(data)
+            return p, False
         except Exception as ex:
             ex_second = None
             try:
                 p = LogGroupListRaw()
                 p.ParseFromString(data)
-                self.loggroup_list = p
-                self._is_bytes_type = True
-                return
+                return p, True
             except Exception as ex2:
                 ex_second = ex2
 

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.9.32'
+__version__ = '0.9.34'
 
 import sys
 OS_VERSION = str(sys.platform)

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34/aliyun_log_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.32
+Version: 0.9.34
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/aliyun_log_python_sdk.egg-info/SOURCES.txt

@@ -95,6 +95,7 @@ aliyun/log/consumer/consumer_client.py
 aliyun/log/consumer/exceptions.py
 aliyun/log/consumer/fetched_log_group.py
 aliyun/log/consumer/heart_beat.py
+aliyun/log/consumer/preprocessor.py
 aliyun/log/consumer/shard_worker.py
 aliyun/log/consumer/tasks.py
 aliyun/log/consumer/worker.py

{aliyun_log_python_sdk-0.9.32 → aliyun_log_python_sdk-0.9.34}/setup.py

@@ -43,7 +43,7 @@ requirements_py2 = [
     'jmespath==0.9.5',
     'enum34==1.1.10',
     'futures==3.3.0',
-    'protobuf>3.4.0,<=3.17.3', # 3.18.0 is the last version support py2, but yanked
+    'protobuf==3.17.3', # 3.18.0 is the last version support py2, but yanked
     'regex==2021.3.17',
     'tzlocal==2.0.0',
     'lz4==2.2.1',