aliyun-log-python-sdk 0.9.33__tar.gz → 0.9.35__tar.gz

{aliyun_log_python_sdk-0.9.33/aliyun_log_python_sdk.egg-info → aliyun_log_python_sdk-0.9.35}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: aliyun-log-python-sdk
-Version: 0.9.33
+Version: 0.9.35
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
@@ -29,6 +29,14 @@ Requires-Dist: pytest; extra == "test"
 Requires-Dist: lz4; extra == "test"
 Requires-Dist: virtualenv; extra == "test"
 Requires-Dist: zstandard; extra == "test"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
 
 
 Python SDK for Alicloud Log Service

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/__init__.py

@@ -7,6 +7,8 @@ from .index_config import IndexConfig, IndexKeyConfig, IndexLineConfig
 from .listtopicsrequest import ListTopicsRequest
 from .listlogstoresrequest import ListLogstoresRequest
 from .logtail_config_detail import *
+from .logtail_pipeline_config_detail import *
+from .logtail_pipeline_config_response import *
 from .machine_group_detail import MachineGroupDetail
 from .putlogsrequest import PutLogsRequest
 from .shipper_config import ShipperTask, OssShipperConfig, OdpsShipperConfig

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/consumer/__init__.py

@@ -1,4 +1,4 @@
 
-from .tasks import ConsumerProcessorBase, ConsumerProcessorAdaptor
+from .tasks import ConsumerProcessorBase, ConsumerProcessorAdaptor, ConsumerJsonProcessorBase
 from .config import LogHubConfig, CursorPosition
 from .worker import ConsumerWorker

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/consumer/consumer_client.py

@@ -7,6 +7,7 @@ from .exceptions import ClientWorkerException
 from ..logexception import LogException
 from ..version import USER_AGENT
 from ..auth import *
+from ..pulllog_response import PullLogResponse
 
 
 class ConsumerClientLoggerAdapter(logging.LoggerAdapter):
@@ -180,4 +181,5 @@ class ConsumerClient(object):
         return self.mclient.get_end_cursor(self.mproject, self.mlogstore, shard_id).get_cursor()
 
     def pull_logs(self, shard_id, cursor, end_cursor=None, count=1000, query=None, processor=None):
+        # type: (int, str, str, int, str, object) -> PullLogResponse
         return self.mclient.pull_logs(self.mproject, self.mlogstore, shard_id, cursor, count, compress=True, end_cursor=end_cursor, query=query, accept_compress_type=self.accept_compress_type, processor=processor)

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/consumer/fetched_log_group.py

@@ -3,18 +3,19 @@
 
 class FetchedLogGroup(object):
 
-    def __init__(self, shard_id, log_group_list, end_cursor):
+    def __init__(self, shard_id, data, end_cursor, log_group_count):
         self._shard_id = shard_id
-        self._fetched_log_group_list = log_group_list
+        self._data = data
         self._end_cursor = end_cursor
+        self._log_group_count = log_group_count
 
     @property
     def shard_id(self):
         return self._shard_id
 
     @property
-    def fetched_log_group_list(self):
-        return self._fetched_log_group_list
+    def data(self):
+        return self._data
 
     @property
     def end_cursor(self):
@@ -22,4 +23,4 @@ class FetchedLogGroup(object):
 
     @property
     def log_group_size(self):
-        return len(self._fetched_log_group_list.LogGroups)
+        return self._log_group_count

aliyun_log_python_sdk-0.9.35/aliyun/log/consumer/preprocessor.py

@@ -0,0 +1,6 @@
+def to_log_group_list_pb(pull_log_response):
+    return pull_log_response.get_loggroup_list()
+
+
+def to_flattern_json_list(pull_log_response):
+    return pull_log_response.get_flatten_logs_json()

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/consumer/shard_worker.py

@@ -66,6 +66,7 @@ class ShardConsumerWorker(object):
         self.save_last_checkpoint = False
         self.query = query
         self.consume_processor = consume_processor
+        self.preprocessor = self.processor._preprocessor
         self.logger = ShardConsumerWorkerLoggerAdapter(
             logging.getLogger(__name__), {"shard_consumer_worker": self})
 
@@ -98,10 +99,13 @@ class ShardConsumerWorker(object):
 
                 self.last_success_fetch_time = time.time()
 
-                self.last_fetch_log_group = FetchedLogGroup(self.shard_id, task_result.get_fetched_log_group_list(),
-                                                            task_result.get_cursor())
+                self.last_fetch_log_group = FetchedLogGroup(self.shard_id,
+                                                            task_result.get_data(),
+                                                            task_result.get_cursor(),
+                                                            task_result.get_log_group_count())
+
                 self.next_fetch_cursor = task_result.get_cursor()
-                self.last_fetch_count = self.last_fetch_log_group.log_group_size
+                self.last_fetch_count = task_result.get_log_group_count()
                 self.last_fetch_size = task_result.get_raw_size()
                 self.rawLogGroupCountBeforeQuery = task_result.get_raw_log_group_count_before_query()
                 self.rawSizeBeforeQuery = task_result.get_raw_size_before_query()
@@ -114,7 +118,6 @@ class ShardConsumerWorker(object):
                         self.checkpoint_tracker.flush_check_point()
                         self.save_last_checkpoint = True
 
-
             self._sample_log_error(task_result)
 
             # no task or task is done, create new task
@@ -136,12 +139,17 @@ class ShardConsumerWorker(object):
                     is_generate_fetch_task = (time.time() - self.last_fetch_time) > 0.05
                 if is_generate_fetch_task:
                     self.last_fetch_time = time.time()
-                    self.fetch_data_future = \
-                        self.executor.submit(consumer_fetch_task,
-                                             self.log_client, self.shard_id, self.next_fetch_cursor,
-                                             max_fetch_log_group_size=self.max_fetch_log_group_size,
-                                             end_cursor=self.fetch_end_cursor, query=self.query,
-                                             consume_processor=self.consume_processor)
+                    self.fetch_data_future = self.executor.submit(
+                        consumer_fetch_task,
+                        self.log_client,
+                        self.preprocessor,
+                        self.shard_id,
+                        self.next_fetch_cursor,
+                        max_fetch_log_group_size=self.max_fetch_log_group_size,
+                        end_cursor=self.fetch_end_cursor,
+                        query=self.query,
+                        consume_processor=self.consume_processor,
+                    )
                 else:
                     self.fetch_data_future = None
             else:
@@ -213,13 +221,12 @@ class ShardConsumerWorker(object):
                 self.checkpoint_tracker.set_cursor(self.last_fetch_log_group.end_cursor)
                 self.current_task_exist = True
 
-                # must deep copy cause some revision will happen
-                last_fetch_log_group = copy.deepcopy(self.last_fetch_log_group)
+                log_group = self.last_fetch_log_group
                 self.last_fetch_log_group = None
 
                 if self.last_fetch_count > 0:
                     self.task_future = self.executor.submit(consumer_process_task, self.processor,
-                                                            last_fetch_log_group.fetched_log_group_list,
+                                                            log_group.data,
                                                             self.checkpoint_tracker)
 
         elif self.consumer_status == ConsumerStatus.SHUTTING_DOWN:

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/consumer/tasks.py

@@ -5,6 +5,9 @@ import logging
 
 from .config import CursorPosition
 from ..logexception import LogException
+from ..pulllog_response import PullLogResponse
+from .consumer_client import ConsumerClient
+from .preprocessor import to_log_group_list_pb, to_flattern_json_list
 import time
 import six
 import sys
@@ -17,6 +20,7 @@ class ConsumerProcessorBase(object):
         self.shard_id = -1
         self.last_check_time = 0
         self.checkpoint_timeout = 3
+        self._preprocessor = to_log_group_list_pb
 
     def save_checkpoint(self, tracker, force=False):
         current_time = time.time()
@@ -53,6 +57,20 @@ class ConsumerProcessorBase(object):
         self.save_checkpoint(check_point_tracker, force=True)
 
 
+class ConsumerJsonProcessorBase(ConsumerProcessorBase):
+
+    def __init__(self):
+        super(ConsumerJsonProcessorBase, self).__init__()
+        self._preprocessor = to_flattern_json_list
+
+    @abc.abstractmethod
+    def process(self, flattern_json_list, check_point_tracker):
+        """
+        @flattern_json_list a list of dict, each dict is a log, eg: [{"__time__": 1761721979, "key1": "value1", "hello": "world", "__tag__:tagkey1": "tagvalue1" }]
+        """
+        raise NotImplementedError("not create method process")
+
+
 class ConsumerProcessorAdaptor(ConsumerProcessorBase):
     def __init__(self, func):
         super(ConsumerProcessorAdaptor, self).__init__()
@@ -112,28 +130,32 @@ class InitTaskResult(TaskResult):
 
 
 class FetchTaskResult(TaskResult):
-    def __init__(self, fetched_log_group_list, cursor, raw_size , raw_size_before_query, raw_log_group_count_before_query):
+    def __init__(self, data, log_group_count, cursor, raw_size , raw_size_before_query, raw_log_group_count_before_query):
         super(FetchTaskResult, self).__init__(None)
-        self.fetched_log_group_list = fetched_log_group_list
-        self.cursor = cursor
-        self.raw_size = raw_size
-        self.raw_size_before_query = raw_size_before_query
-        self.raw_log_group_count_before_query = raw_log_group_count_before_query
+        self._data = data
+        self._log_group_count = log_group_count
+        self._cursor = cursor
+        self._raw_size = raw_size
+        self._raw_size_before_query = raw_size_before_query
+        self._raw_log_group_count_before_query = raw_log_group_count_before_query
 
-    def get_fetched_log_group_list(self):
-        return self.fetched_log_group_list
+    def get_data(self):
+        return self._data
 
     def get_cursor(self):
-        return self.cursor
+        return self._cursor
+
+    def get_log_group_count(self):
+        return self._log_group_count
 
     def get_raw_size(self):
-        return self.raw_size
+        return self._raw_size
 
     def get_raw_size_before_query(self):
-        return self.raw_size_before_query
+        return self._raw_size_before_query
 
     def get_raw_log_group_count_before_query(self):
-        return self.raw_log_group_count_before_query
+        return self._raw_log_group_count_before_query
 
 
 def consumer_process_task(processor, log_groups, check_point_tracker):
@@ -186,16 +208,18 @@ def consumer_initialize_task(processor, consumer_client, shard_id, cursor_positi
     except Exception as e:
         return TaskResult(e)
 
-
-def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None, consume_processor=None):
+def consumer_fetch_task(loghub_client_adapter, preprocessor, shard_id, cursor, max_fetch_log_group_size=1000, end_cursor=None, query=None, consume_processor=None):
+    # type: (ConsumerClient, object, int, str, int, str, str, str) -> FetchTaskResult
     exception = None
 
     for retry_times in range(3):
         try:
+            # type: PullLogResponse
             response = loghub_client_adapter.pull_logs(shard_id, cursor, count=max_fetch_log_group_size, end_cursor=end_cursor, query=query, processor=consume_processor)
-            fetch_log_group_list = response.get_loggroup_list()
+            data = preprocessor(response)
             next_cursor = response.get_next_cursor()
             raw_size = response.get_raw_size()
+            log_group_count = response.get_loggroup_count()
             raw_size_before_query = 0
             raw_log_group_count_before_query = 0
             if query or consume_processor:
@@ -203,11 +227,11 @@ def consumer_fetch_task(loghub_client_adapter, shard_id, cursor, max_fetch_log_g
                 raw_log_group_count_before_query = max(response.get_raw_log_group_count_before_query(), 0)
             logger.debug("shard id = %s cursor = %s next cursor = %s size: %s",
                          shard_id, cursor, next_cursor,
-                         response.get_log_count())
+                         log_group_count)
             if not next_cursor:
-                return FetchTaskResult(fetch_log_group_list, cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
+                return FetchTaskResult(data, log_group_count, cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
             else:
-                return FetchTaskResult(fetch_log_group_list, next_cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
+                return FetchTaskResult(data, log_group_count, next_cursor, raw_size, raw_size_before_query, raw_log_group_count_before_query)
         except LogException as e:
             exception = e
             if exception.get_resp_status() == 403:

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/etl_core/trans_comp/trans_kv.py

@@ -32,12 +32,14 @@ class kv_transformer(trans_comp_check_mdoe_base):
     @staticmethod
     @cached
     def _get_kv_ptn(sep, quote, escape):
-        p1 = u'(?!{0})([\u4e00-\u9fa5\u0800-\u4e00\\w\\.\\-]+)\\s*{0}\\s*([\u4e00-\u9fa5\u0800-\u4e00\\w\\.\\-]+)'
+        # Use Unicode characters directly to avoid escape sequence warnings
+        # \u4e00-\u9fa5 = Chinese characters, \u0800-\u4e00 = other Unicode range
+        p1 = r'(?!{0})([' + '\u4e00-\u9fa5\u0800-\u4e00' + r'\w\.\-]+)\s*{0}\s*([' + '\u4e00-\u9fa5\u0800-\u4e00' + r'\w\.\-]+)'
         if not escape:
-            p2 = u'(?!{0})([\u4e00-\u9fa5\u0800-\u4e00\\w\\.\\-]+)\\s*{0}\\s*{1}\s*([^{1}]*?)\s*{1}'
+            p2 = r'(?!{0})([' + '\u4e00-\u9fa5\u0800-\u4e00' + r'\w\.\-]+)\s*{0}\s*{1}\s*([^{1}]*?)\s*{1}'
         else:
-            p2 = u'(?!{0})([\u4e00-\u9fa5\u0800-\u4e00\\w\\.\\-]+)\\s*{0}\\s*{1}\s*((?:[^{1}]|\\\\{1})*?[^\\\\]){1}'
-        ps = u'|'.join([p1, p2]).format(sep, quote)
+            p2 = r'(?!{0})([' + '\u4e00-\u9fa5\u0800-\u4e00' + r'\w\.\-]+)\s*{0}\s*{1}\s*((?:[^{1}]|\\{1})*?[^\\]){1}'
+        ps = '|'.join([p1, p2]).format(sep, quote)
 
         logger.info(u"trans_comp_kv: get ptn: {0}".format(ps))
         return re.compile(ps)

{aliyun_log_python_sdk-0.9.33 → aliyun_log_python_sdk-0.9.35}/aliyun/log/logclient.py

@@ -41,6 +41,7 @@ from .logexception import LogException
 from .logstore_config_response import *
 from .substore_config_response import *
 from .logtail_config_response import *
+from .logtail_pipeline_config_response import *
 from .machinegroup_response import *
 from .rebuild_index_response import *
 from .project_response import *
@@ -2328,6 +2329,132 @@ class LogClient(object):
         (resp, header) = self._send("GET", project_name, None, resource, params, headers)
         return ListLogtailConfigResponse(resp, header)
 
+    def create_logtail_pipeline_config(self, project_name, config_detail):
+        """ create logtail pipeline config in a project
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type config_detail: LogtailPipelineConfigDetail
+        :param config_detail: the logtail pipeline config detail
+
+        :return: CreateLogtailPipelineConfigResponse
+
+        :raise: LogException
+        """
+        headers = {}
+        params = {}
+        resource = "/pipelineconfigs"
+        headers['Content-Type'] = 'application/json'
+        body = six.b(json.dumps(config_detail.to_json()))
+        headers['x-log-bodyrawsize'] = str(len(body))
+        (resp, headers) = self._send("POST", project_name, body, resource, params, headers)
+        return CreateLogtailPipelineConfigResponse(headers, resp)
+
+    def update_logtail_pipeline_config(self, project_name, config_detail):
+        """ update logtail pipeline config in a project
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type config_detail: LogtailPipelineConfigDetail
+        :param config_detail: the logtail pipeline config detail
+
+        :return: UpdateLogtailPipelineConfigResponse
+
+        :raise: LogException
+        """
+        headers = {}
+        params = {}
+        resource = "/pipelineconfigs/" + config_detail.config_name
+        headers['Content-Type'] = 'application/json'
+        body = six.b(json.dumps(config_detail.to_json()))
+        headers['x-log-bodyrawsize'] = str(len(body))
+        (resp, headers) = self._send("PUT", project_name, body, resource, params, headers)
+        return UpdateLogtailPipelineConfigResponse(headers, resp)
+
+    def delete_logtail_pipeline_config(self, project_name, config_name):
+        """ delete logtail pipeline config in a project
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type config_name: string
+        :param config_name: the logtail pipeline config name
+
+        :return: DeleteLogtailPipelineConfigResponse
+
+        :raise: LogException
+        """
+        headers = {}
+        params = {}
+        resource = "/pipelineconfigs/" + config_name
+        (resp, headers) = self._send("DELETE", project_name, None, resource, params, headers)
+        return DeleteLogtailPipelineConfigResponse(headers, resp)
+
+    def get_logtail_pipeline_config(self, project_name, config_name):
+        """ get logtail pipeline config in a project
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type config_name: string
+        :param config_name: the logtail pipeline config name
+
+        :return: GetLogtailPipelineConfigResponse
+
+        :raise: LogException
+        """
+        headers = {}
+        params = {}
+        resource = "/pipelineconfigs/" + config_name
+        (resp, headers) = self._send("GET", project_name, None, resource, params, headers)
+        return GetLogtailPipelineConfigResponse(resp, headers)
+
+    def list_logtail_pipeline_config(self, project_name, config_name=None, logstore_name=None, offset=0, size=100):
+        """ list logtail pipeline config names in a project
+        Unsuccessful operation will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type config_name: string
+        :param config_name: config name to filter
+
+        :type logstore_name: string
+        :param logstore_name: logstore name to filter
+
+        :type offset: int
+        :param offset: the offset of all config names
+
+        :type size: int
+        :param size: the max return names count, -1 means all
+
+        :return: ListLogtailPipelineConfigResponse
+
+        :raise: LogException
+        """
+        # need to use extended method to get more
+        if int(size) == -1 or int(size) > MAX_LIST_PAGING_SIZE:
+            return list_more(self.list_logtail_pipeline_config, int(offset), int(size), MAX_LIST_PAGING_SIZE,
+                           project_name, config_name, logstore_name)
+
+        headers = {}
+        params = {}
+        resource = "/pipelineconfigs"
+        params['offset'] = str(offset)
+        params['size'] = str(size)
+        if config_name:
+            params['configName'] = config_name
+        if logstore_name:
+            params['logstoreName'] = logstore_name
+        (resp, header) = self._send("GET", project_name, None, resource, params, headers)
+        return ListLogtailPipelineConfigResponse(resp, header)
+
     def create_machine_group(self, project_name, group_detail):
         """ create machine group in a project
         Unsuccessful operation will cause an LogException.

aliyun_log_python_sdk-0.9.35/aliyun/log/logtail_pipeline_config_detail.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+"""
+Logtail Pipeline Config Detail Classes
+
+This module provides classes for managing Logtail pipeline configurations.
+"""
+
+from __future__ import print_function
+import json
+
+__all__ = ['LogtailPipelineConfigDetail']
+
+
+class LogtailPipelineConfigDetail(object):
+    """Logtail Pipeline Config Detail
+    
+    :type config_name: string
+    :param config_name: config name, can only contain lowercase letters, numbers, - and _, 
+                        must start and end with lowercase letter or number, length between 2-128 characters
+    
+    :type log_sample: string
+    :param log_sample: log sample, supports multiple log entries
+    
+    :type global_config: dict
+    :param global_config: global configuration
+    
+    :type inputs: list
+    :param inputs: input plugin list, currently only allows 1 input plugin
+    
+    :type processors: list
+    :param processors: processor plugin list
+    
+    :type aggregators: list
+    :param aggregators: aggregator plugin list, maximum 1 aggregator plugin allowed
+    
+    :type flushers: list
+    :param flushers: flusher plugin list, currently only allows 1 flusher_sls plugin
+    """
+    
+    def __init__(self, config_name, inputs, flushers, log_sample=None, 
+                 global_config=None, processors=None, aggregators=None):
+        """Initialize a Logtail Pipeline Config Detail
+        
+        :type config_name: string
+        :param config_name: config name
+        
+        :type inputs: list
+        :param inputs: input plugin list
+        
+        :type flushers: list
+        :param flushers: flusher plugin list
+        
+        :type log_sample: string
+        :param log_sample: log sample (optional)
+        
+        :type global_config: dict
+        :param global_config: global configuration (optional)
+        
+        :type processors: list
+        :param processors: processor plugin list (optional)
+        
+        :type aggregators: list
+        :param aggregators: aggregator plugin list (optional)
+        """
+        self.config_name = config_name
+        self.log_sample = log_sample
+        self.global_config = global_config if global_config is not None else {}
+        self.inputs = inputs if inputs is not None else []
+        self.processors = processors if processors is not None else []
+        self.aggregators = aggregators if aggregators is not None else []
+        self.flushers = flushers if flushers is not None else []
+    
+    def to_json(self):
+        """Convert to JSON object
+        
+        :return: dict object ready for JSON serialization
+        """
+        json_value = {
+            "configName": self.config_name
+        }
+        
+        # Add optional fields
+        if self.log_sample:
+            json_value["logSample"] = self.log_sample
+        
+        if self.global_config:
+            json_value["global"] = self.global_config
+        
+        # Required fields
+        json_value["inputs"] = self.inputs
+        json_value["flushers"] = self.flushers
+        
+        # Optional fields
+        if self.processors:
+            json_value["processors"] = self.processors
+        
+        if self.aggregators:
+            json_value["aggregators"] = self.aggregators
+        
+        return json_value
+    
+    @staticmethod
+    def from_json(json_value):
+        """Create LogtailPipelineConfigDetail from JSON object
+        
+        :type json_value: dict
+        :param json_value: JSON object
+        
+        :return: LogtailPipelineConfigDetail object
+        """
+        config_name = json_value.get("configName", "")
+        log_sample = json_value.get("logSample", None)
+        global_config = json_value.get("global", None)
+        inputs = json_value.get("inputs", [])
+        processors = json_value.get("processors", None)
+        aggregators = json_value.get("aggregators", None)
+        flushers = json_value.get("flushers", [])
+        
+        return LogtailPipelineConfigDetail(
+            config_name=config_name,
+            inputs=inputs,
+            flushers=flushers,
+            log_sample=log_sample,
+            global_config=global_config,
+            processors=processors,
+            aggregators=aggregators
+        )
+
+
+