aliyun-log-python-sdk 0.8.15__tar.gz → 0.9.1__tar.gz

aliyun-log-python-sdk-0.9.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Alibaba Cloud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{aliyun-log-python-sdk-0.8.15 → aliyun-log-python-sdk-0.9.1}/PKG-INFO

@@ -1,16 +1,9 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.8.15
+Version: 0.9.1
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
-Author-email: UNKNOWN
-License: UNKNOWN
-Description: 
-        Python SDK for Alicloud Log Service
-        http://aliyun-log-python-sdk.readthedocs.io
-        
-Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -22,3 +15,15 @@ Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: Implementation :: PyPy
+License-File: LICENSE
+Requires-Dist: six
+Requires-Dist: requests
+Requires-Dist: python-dateutil
+Requires-Dist: elasticsearch
+Requires-Dist: jmespath
+Requires-Dist: dateparser
+Requires-Dist: protobuf<4.0.0,>3.4.0
+
+
+Python SDK for Alicloud Log Service
+http://aliyun-log-python-sdk.readthedocs.io

{aliyun-log-python-sdk-0.8.15 → aliyun-log-python-sdk-0.9.1}/aliyun/log/consumer/config.py

@@ -25,7 +25,7 @@ class LogHubConfig(object):
                  cursor_position=None, heartbeat_interval=None, data_fetch_interval=None, in_order=False,
                  cursor_start_time=None, security_token=None, max_fetch_log_group_size=None, worker_pool_size=None, shared_executor=None,
                  cursor_end_time=None, credentials_refresher=None,
-                 auth_version=AUTH_VERSION_1, region=''):
+                 auth_version=AUTH_VERSION_1, region='', query=None):
         """
 
         :param endpoint:
@@ -47,6 +47,7 @@ class LogHubConfig(object):
         :param cursor_end_time: cursor end time, default is None (never stop processing). could setting it as ISO time-format, when setting it as "end", it means process all logs received from start to the time when the consumer is started.
         :param auth_version: only support AUTH_VERSION_1 and AUTH_VERSION_4
         :param region: region of project
+        :param query: the SPL query, such as *| where a = 'xxx'
         """
         self.endpoint = endpoint
         self.accessKeyId = access_key_id
@@ -69,3 +70,4 @@ class LogHubConfig(object):
         self.credentials_refresher = credentials_refresher
         self.auth_version = auth_version
         self.region = region
+        self.query = query

{aliyun-log-python-sdk-0.8.15 → aliyun-log-python-sdk-0.9.1}/aliyun/log/consumer/consumer_client.py

@@ -27,7 +27,7 @@ class ConsumerClient(object):
         from .. import LogClient
 
         self.mclient = LogClient(endpoint, access_key_id, access_key, security_token, auth_version=auth_version, region=region)
-        self.mclient.set_user_agent('%s-consumergroup-%s' % (USER_AGENT, consumer_group))
+        self.mclient.set_user_agent('%s-consumergroup-%s-%s' % (USER_AGENT, consumer_group, consumer))
         if credentials_refresher is not None:
             self.mclient.set_credentials_auto_refresher(credentials_refresher)
         self.mproject = project
@@ -100,5 +100,5 @@ class ConsumerClient(object):
     def get_end_cursor(self, shard_id):
         return self.mclient.get_end_cursor(self.mproject, self.mlogstore, shard_id).get_cursor()
 
-    def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000):
-        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor)
+    def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None):
+        return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor,query=query)

{aliyun-log-python-sdk-0.8.15 → aliyun-log-python-sdk-0.9.1}/aliyun/log/consumer/shard_worker.py

@@ -30,7 +30,7 @@ class ShardConsumerWorkerLoggerAdapter(logging.LoggerAdapter):
 
 class ShardConsumerWorker(object):
     def __init__(self, log_client, shard_id, consumer_name, processor, cursor_position, cursor_start_time,
-                 max_fetch_log_group_size=1000, executor=None, cursor_end_time=None):
+                 max_fetch_log_group_size=1000, executor=None, cursor_end_time=None, query=None):
         self.log_client = log_client
         self.shard_id = shard_id
         self.consumer_name = consumer_name
@@ -57,10 +57,13 @@ class ShardConsumerWorker(object):
         self.last_log_error_time = 0
         self.last_fetch_time = 0
         self.last_fetch_count = 0
+        self.last_fetch_size = 0
+        self.rawLogGroupCountBeforeQuery = 0
+        self.rawSizeBeforeQuery = 0
         self.last_success_fetch_time = 0
         self.last_success_fetch_time_with_data = 0
         self.save_last_checkpoint = False
-
+        self.query = query
         self.logger = ShardConsumerWorkerLoggerAdapter(
             logging.getLogger(__name__), {"shard_consumer_worker": self})
 
@@ -97,6 +100,9 @@ class ShardConsumerWorker(object):
                                                             task_result.get_cursor())
                 self.next_fetch_cursor = task_result.get_cursor()
                 self.last_fetch_count = self.last_fetch_log_group.log_group_size
+                self.last_fetch_size = task_result.get_raw_size()
+                self.rawLogGroupCountBeforeQuery = task_result.get_raw_log_group_count_before_query()
+                self.rawSizeBeforeQuery = task_result.get_raw_size_before_query()
                 if self.last_fetch_count > 0:
                     self.last_success_fetch_time_with_data = time.time()
                     self.save_last_checkpoint = False
@@ -113,22 +119,26 @@ class ShardConsumerWorker(object):
             if task_result is None or task_result.get_exception() is None:
                 # flag to indicate if it's done
                 is_generate_fetch_task = True
+                fetch_size = self.last_fetch_size
+                fetch_count = self.last_fetch_count
+                if self.query:
+                    fetch_size = self.rawSizeBeforeQuery
+                    fetch_count = self.rawLogGroupCountBeforeQuery
 
                 # throttling control, similar as Java's SDK
-                if self.last_fetch_count < 100:
+                if fetch_size < 1024 * 1024 and fetch_count < 100 and fetch_count < self.max_fetch_log_group_size:
                     is_generate_fetch_task = (time.time() - self.last_fetch_time) > 0.5
-                elif self.last_fetch_count < 500:
+                elif fetch_size < 2 * 1024 * 1024 and fetch_count < 500 and fetch_count < self.max_fetch_log_group_size:
                     is_generate_fetch_task = (time.time() - self.last_fetch_time) > 0.2
-                elif self.last_fetch_count < 1000:
+                elif fetch_size < 4 * 1024 * 1024 and fetch_count < 1000 and fetch_count < self.max_fetch_log_group_size:
                     is_generate_fetch_task = (time.time() - self.last_fetch_time) > 0.05
-
                 if is_generate_fetch_task:
                     self.last_fetch_time = time.time()
                     self.fetch_data_future = \
                         self.executor.submit(consumer_fetch_task,
                                              self.log_client, self.shard_id, self.next_fetch_cursor,
                                              max_fetch_log_group_size=self.max_fetch_log_group_size,
-                                             end_cursor=self.fetch_end_cursor)
+                                             end_cursor=self.fetch_end_cursor, query=self.query)
                 else:
                     self.fetch_data_future = None
             else:

{aliyun-log-python-sdk-0.8.15 → aliyun-log-python-sdk-0.9.1}/aliyun/log/consumer/tasks.py

@@ -112,10 +112,13 @@ class InitTaskResult(TaskResult):
 
 
 class FetchTaskResult(TaskResult):
-    def __init__(self, fetched_log_group_list, cursor):
+    def __init__(self, fetched_log_group_list, cursor, raw_size , raw_size_before_query, raw_log_group_count_before_query):
         super(FetchTaskResult, self).__init__(None)
         self.fetched_log_group_list = fetched_log_group_list
         self.cursor = cursor
+        self.raw_size = raw_size
+        self.raw_size_before_query = raw_size_before_query
+        self.raw_log_group_count_before_query = raw_log_group_count_before_query
 
     def get_fetched_log_group_list(self):
         return self.fetched_log_group_list
@@ -123,6 +126,15 @@ class FetchTaskResult(TaskResult):
     def get_cursor(self):
         return self.cursor
 
+    def get_raw_size(self):
+        return self.raw_size
+
+    def get_raw_size_before_query(self):
+        return self.raw_size_before_query
+
+    def get_raw_log_group_count_before_query(self):
+        return self.raw_log_group_count_before_query
+
 
 def consumer_process_task(processor, log_groups, check_point_tracker):
     """
@@ -175,23 +187,31 @@ def consumer_initialize_task(processor, consumer_client, shard_id, cursor_positi
         return TaskResult(e)
 
 
-def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None):
+def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None):
     exception = None
 
     for retry_times in range(3):
         try:
-            response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor)
+            response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query)
             fetch_log_group_list = response.get_loggroup_list()
             next_cursor = response.get_next_cursor()
+            raw_size = response.get_raw_size()
+            raw_size_before_query = 0
+            raw_log_group_count_before_query = 0
+            if query:
+                raw_size_before_query = max(response.get_raw_size_before_query(), 0)
+                raw_log_group_count_before_query = max(response.get_raw_log_group_count_before_query(), 0)
             logger.debug("shard id = %s cursor = %s next cursor = %s size: %s",
                          shard_id, cursor, next_cursor,
                          response.get_log_count())
             if not next_cursor:
-                return FetchTaskResult(fetch_log_group_list, cursor)
+                return FetchTaskResult(fetch_log_group_list, cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
             else:
-                return FetchTaskResult(fetch_log_group_list, next_cursor)
+                return FetchTaskResult(fetch_log_group_list, next_cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
         except LogException as e:
             exception = e
+            if exception.get_resp_status() == 403:
+                time.sleep(5)
         except Exception as e1:
             logger.error(e1, exc_info=True)
             raise Exception(e1)

{aliyun-log-python-sdk-0.8.15 → aliyun-log-python-sdk-0.9.1}/aliyun/log/consumer/worker.py

@@ -196,6 +196,7 @@ class ConsumerWorker(Thread):
                                        self.option.cursor_position, self.option.cursor_start_time,
                                        executor=self._executor,
                                        cursor_end_time=self.option.cursor_end_time,
-                                       max_fetch_log_group_size=self.option.max_fetch_log_group_size)
+                                       max_fetch_log_group_size=self.option.max_fetch_log_group_size,
+                                       query=self.option.query)
         self.shard_consumers[shard_id] = consumer
         return consumer