aliyun-log-python-sdk 0.9.26__tar.gz → 0.9.28__tar.gz

{aliyun_log_python_sdk-0.9.26/aliyun_log_python_sdk.egg-info → aliyun_log_python_sdk-0.9.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.26
+Version: 0.9.28
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun

{aliyun_log_python_sdk-0.9.26 → aliyun_log_python_sdk-0.9.28}/aliyun/log/__init__.py

@@ -35,7 +35,12 @@ from .common_response import *
 from .external_store_config_response import *
 from .proto import LogGroupRaw as LogGroup
 from .rebuild_index_response import *
-
+from .deletelogsrequest import *
+from .deletelogssresponse import *
+from .getdeletelogsstatusrequest import *
+from .getdeletelogsstatusresponse import *
+from .listdeletelogsstasksrequest import *
+from .listdeletelogsstasksresponse import *
 # logging handler
 from .logger_hanlder import SimpleLogHandler, QueuedLogHandler, LogFields, UwsgiQueuedLogHandler
 from .metering_mode_response import GetLogStoreMeteringModeResponse, \

{aliyun_log_python_sdk-0.9.26 → aliyun_log_python_sdk-0.9.28}/aliyun/log/consumer/config.py

@@ -26,7 +26,7 @@ class LogHubConfig(object):
                  cursor_start_time=None, security_token=None, max_fetch_log_group_size=None, worker_pool_size=None, shared_executor=None,
                  cursor_end_time=None, credentials_refresher=None,
                  auth_version=AUTH_VERSION_1, region='', query=None,
-                 accept_compress_type=None):
+                 accept_compress_type=None, processor=None):
         """
 
         :param endpoint:
@@ -49,6 +49,7 @@ class LogHubConfig(object):
         :param auth_version: only support AUTH_VERSION_1 and AUTH_VERSION_4
         :param region: region of project
         :param query: the SPL query, such as *| where a = 'xxx'
+        :param processor: the consume processor which contains SPL, such as consume-processor-1
         :type accept_compress_type: string
         :param accept_compress_type: The compression type used for logs retrieved from sls.
         Supported types include 'lz4' and 'zstd'. If you choose 'zstd', ensure the `zstd` library is installed via pip. The default value is 'lz4'.
@@ -77,3 +78,4 @@ class LogHubConfig(object):
         self.region = region
         self.query = query
         self.accept_compress_type = accept_compress_type
+        self.processor = processor

{aliyun_log_python_sdk-0.9.26 → aliyun_log_python_sdk-0.9.28}/aliyun/log/consumer/consumer_client.py

@@ -179,5 +179,5 @@ class ConsumerClient(object):
     def get_end_cursor(self, shard_id):
         return self.mclient.get_end_cursor(self.mproject, self.mlogstore, shard_id).get_cursor()
 
-    def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None):
-        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor, query=query, accept_compress_type=self.accept_compress_type)
+    def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None, processor=None):
+        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor, query=query, accept_compress_type=self.accept_compress_type, processor=processor)

{aliyun_log_python_sdk-0.9.26 → aliyun_log_python_sdk-0.9.28}/aliyun/log/consumer/shard_worker.py

@@ -2,6 +2,7 @@
 
 import copy
 import logging
+from platform import processor
 import time
 
 from .checkpoint_tracker import ConsumerCheckpointTracker
@@ -30,7 +31,7 @@ class ShardConsumerWorkerLoggerAdapter(logging.LoggerAdapter):
 
 class ShardConsumerWorker(object):
     def __init__(self, log_client, shard_id, consumer_name, processor, cursor_position, cursor_start_time,
-                 max_fetch_log_group_size=1000, executor=None, cursor_end_time=None, query=None):
+                 max_fetch_log_group_size=1000, executor=None, cursor_end_time=None, query=None, consume_processor=None):
         self.log_client = log_client
         self.shard_id = shard_id
         self.consumer_name = consumer_name
@@ -64,6 +65,7 @@ class ShardConsumerWorker(object):
         self.last_success_fetch_time_with_data = 0
         self.save_last_checkpoint = False
         self.query = query
+        self.consume_processor = consume_processor
         self.logger = ShardConsumerWorkerLoggerAdapter(
             logging.getLogger(__name__), {"shard_consumer_worker": self})
 
@@ -121,7 +123,7 @@ class ShardConsumerWorker(object):
                 is_generate_fetch_task = True
                 fetch_size = self.last_fetch_size
                 fetch_count = self.last_fetch_count
-                if self.query:
+                if self.query or self.consume_processor:
                     fetch_size = self.rawSizeBeforeQuery
                     fetch_count = self.rawLogGroupCountBeforeQuery
 
@@ -138,7 +140,8 @@ class ShardConsumerWorker(object):
                         self.executor.submit(consumer_fetch_task,
                                              self.log_client, self.shard_id, self.next_fetch_cursor,
                                              max_fetch_log_group_size=self.max_fetch_log_group_size,
-                                             end_cursor=self.fetch_end_cursor, query=self.query)
+                                             end_cursor=self.fetch_end_cursor, query=self.query,
+                                             consume_processor=self.consume_processor)
                 else:
                     self.fetch_data_future = None
             else:

{aliyun_log_python_sdk-0.9.26 → aliyun_log_python_sdk-0.9.28}/aliyun/log/consumer/tasks.py

@@ -187,18 +187,18 @@ def consumer_initialize_task(processor, consumer_client, shard_id, cursor_positi
         return TaskResult(e)
 
 
-def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None):
+def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None, consume_processor=None):
     exception = None
 
     for retry_times in range(3):
         try:
-            response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query)
+            response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query, processor=consume_processor)
             fetch_log_group_list = response.get_loggroup_list()
             next_cursor = response.get_next_cursor()
             raw_size = response.get_raw_size()
             raw_size_before_query = 0
             raw_log_group_count_before_query = 0
-            if query:
+            if query or consume_processor:
                 raw_size_before_query = max(response.get_raw_size_before_query(), 0)
                 raw_log_group_count_before_query = max(response.get_raw_log_group_count_before_query(), 0)
             logger.debug("shard id = %s cursor = %s next cursor = %s size: %s",

{aliyun_log_python_sdk-0.9.26 → aliyun_log_python_sdk-0.9.28}/aliyun/log/consumer/worker.py

@@ -198,6 +198,7 @@ class ConsumerWorker(Thread):
                                        executor=self._executor,
                                        cursor_end_time=self.option.cursor_end_time,
                                        max_fetch_log_group_size=self.option.max_fetch_log_group_size,
-                                       query=self.option.query)
+                                       query=self.option.query,
+                                       consume_processor=self.option.processor)
         self.shard_consumers[shard_id] = consumer
         return consumer

aliyun_log_python_sdk-0.9.28/aliyun/log/deletelogsrequest.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logrequest import LogRequest
+from .util import parse_timestamp
+
+class DeleteLogsRequest(LogRequest):
+    """ The request used to get histograms of a query from log.
+
+    :type project: string
+    :param project: project name
+
+    :type logstore: string
+    :param logstore: logstore name
+
+    :type fromTime: int/string
+    :param fromTime: the begin time or format of time in readable time like "%Y-%m-%d %H:%M:%S<time_zone>" e.g. "2018-01-02 12:12:10+8:00" e.g. "2018-01-02 12:12:10", also support human readable string, e.g. "1 hour ago", "now", "yesterday 0:0:0", refer to https://aliyun-log-cli.readthedocs.io/en/latest/tutorials/tutorial_human_readable_datetime.html
+
+    :type toTime: int/string
+    :param toTime: the end time or format of time in readable time like "%Y-%m-%d %H:%M:%S<time_zone>" e.g. "2018-01-02 12:12:10+8:00" e.g. "2018-01-02 12:12:10", also support human readable string, e.g. "1 hour ago", "now", "yesterday 0:0:0", refer to https://aliyun-log-cli.readthedocs.io/en/latest/tutorials/tutorial_human_readable_datetime.html
+
+    :type topic: string
+    :param topic: topic name of logs
+
+    :type query: string
+    :param query: user defined query
+    """
+
+    def __init__(self, project=None, logstore=None, fromTime=None, toTime=None, topic=None, query=None):
+        LogRequest.__init__(self, project)
+        self.logstore = logstore
+        self.fromTime = parse_timestamp(fromTime)
+        self.toTime = parse_timestamp(toTime)
+        self.topic = topic
+        self.query = query
+
+    def get_logstore(self):
+        """ Get logstore name
+
+        :return: string, logstore name.
+        """
+        return self.logstore if self.logstore else ''
+
+    def set_logstore(self, logstore):
+        """ Set logstore name
+
+        :type logstore: string
+        :param logstore: logstore name
+        """
+        self.logstore = logstore
+
+    def get_topic(self):
+        """ Get topic name
+
+        :return: string, topic name
+        """
+        return self.topic if self.topic else ''
+
+    def set_topic(self, topic):
+        """ Set topic name
+
+        :type topic: string
+        :param topic: topic name
+        """
+        self.topic = topic
+
+    def get_from(self):
+        """ Get begin time
+
+        :return: int, begin time
+        """
+        return self.fromTime
+
+    def set_from(self, fromTime):
+        """ Set begin time
+
+        :type fromTime: int
+        :param fromTime: begin time
+        """
+        self.fromTime = fromTime
+
+    def get_to(self):
+        """ Get end time
+
+        :return: int, end time
+        """
+        return self.toTime
+
+    def set_to(self, toTime):
+        """ Set end time
+
+        :type toTime: int
+        :param toTime: end time
+        """
+        self.toTime = toTime
+
+    def get_query(self):
+        """ Get user defined query
+
+        :return: string, user defined query
+        """
+        return self.query
+
+    def set_query(self, query):
+        """ Set user defined query
+
+        :type query: string
+        :param query: user defined query
+        """
+        self.query = query

aliyun_log_python_sdk-0.9.28/aliyun/log/deletelogssresponse.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logresponse import LogResponse
+from .histogram import Histogram
+from .util import Util
+import json
+
+
+class DeleteLogsResponse(LogResponse):
+    """ The response of the DeleteLogs API from log.
+
+    :type resp: dict
+    :param resp: DeleteLogsResponse HTTP response body
+
+    :type header: dict
+    :param header: DeleteLogsResponse HTTP response header
+    """
+    def __init__(self, resp, header):
+        LogResponse.__init__(self, header, resp)
+        self.taskid = resp.get('taskId')
+
+    def get_taskid(self):
+        return self.taskid
+
+    def log_print(self):
+        print('DeleteLogsResponse:')
+        print('headers:', self.get_all_headers())
+        print('taskid:', self.taskid)

aliyun_log_python_sdk-0.9.28/aliyun/log/getdeletelogsstatusrequest.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logrequest import LogRequest
+from .util import parse_timestamp
+
+class GetDeleteLogsStatusRequest(LogRequest):
+    """ The request used to get histograms of a query from log.
+
+    :type project: string
+    :param project: project name
+
+    :type logstore: string
+    :param logstore: logstore name
+
+    :type taskid: string
+    :param taskid: delete query taskid
+
+    """
+
+    def __init__(self, project=None, logstore=None, taskid=None):
+        LogRequest.__init__(self, project)
+        self.logstore = logstore
+        self.taskid = taskid
+    def get_logstore(self):
+        """ Get logstore name
+
+        :return: string, logstore name.
+        """
+        return self.logstore if self.logstore else ''
+
+    def set_logstore(self, logstore):
+        """ Set logstore name
+
+        :type logstore: string
+        :param logstore: logstore name
+        """
+        self.logstore = logstore
+
+    def get_taskid(self):
+        """ Get taskid name
+
+        :return: string, taskid name.
+        """
+        return self.taskid if self.taskid else ''
+
+    def set_taskid(self, taskid):
+        """ Set taskid name
+
+        :type taskid: string
+        :param taskid: taskid name
+        """
+        self.taskid = taskid
+

aliyun_log_python_sdk-0.9.28/aliyun/log/getdeletelogsstatusresponse.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logresponse import LogResponse
+
+class GetDeleteLogsStatusResponse(LogResponse):
+    """ The response of the GetDeleteLogsStatus API from log.
+
+    :type resp: dict
+    :param resp: GetDeleteLogsStatusResponse HTTP response body
+
+    :type header: dict
+    :param header: GetDeleteLogsStatusResponse HTTP response header
+    """
+
+    def __init__(self, resp, header):
+        LogResponse.__init__(self, header, resp)
+        self.process = resp['progress']
+
+    def is_completed(self):
+        """ Check if the histogram is completed
+
+        :return: bool, true if this histogram is completed
+        """
+        return self.process == 100.0
+
+    def get_process(self):
+        return self.process
+
+    def log_print(self):
+        print('GetDeleteLogsStatusResponse:')
+        print('headers:', self.get_all_headers())
+        print('progress:', self.process)
+

aliyun_log_python_sdk-0.9.28/aliyun/log/listdeletelogsstasksrequest.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logrequest import LogRequest
+
+
+class ListDeleteLogsTasksRequest(LogRequest):
+    """ The request used to list delete logs tasks from log.
+
+    :type project: string
+    :param project: project name
+
+    :type logstore: string
+    :param logstore: logstore name
+    """
+
+    def __init__(self, project=None, logstore=None):
+        LogRequest.__init__(self, project)
+        self.logstore = logstore
+
+    def get_logstore(self):
+        """ Get logstore name
+
+        :return: string, logstore name.
+        """
+        return self.logstore if self.logstore else ''
+
+    def set_logstore(self, logstore):
+        """ Set logstore name
+
+        :type logstore: string
+        :param logstore: logstore name
+        """
+        self.logstore = logstore

aliyun_log_python_sdk-0.9.28/aliyun/log/listdeletelogsstasksresponse.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logresponse import LogResponse
+from .util import Util
+import json
+
+
+class ListDeleteLogsTasksResponse(LogResponse):
+    """ The response of the ListDeleteLogsTasks API from log.
+
+    :type resp: dict
+    :param resp: ListDeleteLogsTasksResponse HTTP response body
+
+    :type header: dict
+    :param header: ListDeleteLogsTasksResponse HTTP response header
+    """
+
+    def __init__(self, resp, header):
+        LogResponse.__init__(self, header, resp)
+        self.count = resp.get('count', 0)
+        self.total = resp.get('total', 0)
+        self.task_list = resp.get('tasks', [])
+
+
+    def is_completed(self):
+        """ Check if the request is completed
+
+        :return: bool, true if this request is completed
+        """
+        # 根据实际数据，当所有任务都完成时(进度为100%)表示完成
+        if not self.task_list:
+            return False
+        return all(task.get('progress', 0) == 100 for task in self.task_list)
+
+    def get_task_list(self):
+        return self.task_list
+
+    def get_count(self):
+        return self.count
+
+    def get_total(self):
+        return self.total
+
+    def log_print(self):
+        print('ListDeleteLogsTasksResponse:')
+        print('headers:', self.get_all_headers())
+        print('count:', self.count)
+        print('total:', self.total)
+        print('task_list:', self.task_list)