aliyun-log-python-sdk 0.9.0__tar.gz → 0.9.1__tar.gz

{aliyun-log-python-sdk-0.9.0/aliyun_log_python_sdk.egg-info → aliyun-log-python-sdk-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.0
+Version: 0.9.1
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
@@ -16,6 +16,13 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 License-File: LICENSE
+Requires-Dist: six
+Requires-Dist: requests
+Requires-Dist: python-dateutil
+Requires-Dist: elasticsearch
+Requires-Dist: jmespath
+Requires-Dist: dateparser
+Requires-Dist: protobuf<4.0.0,>3.4.0
 
 
 Python SDK for Alicloud Log Service

{aliyun-log-python-sdk-0.9.0 → aliyun-log-python-sdk-0.9.1}/aliyun/log/consumer/consumer_client.py

@@ -27,7 +27,7 @@ class ConsumerClient(object):
         from .. import LogClient
 
         self.mclient = LogClient(endpoint, access_key_id, access_key, security_token, auth_version=auth_version, region=region)
-        self.mclient.set_user_agent('%s-consumergroup-%s' % (USER_AGENT, consumer_group))
+        self.mclient.set_user_agent('%s-consumergroup-%s-%s' % (USER_AGENT, consumer_group, consumer))
         if credentials_refresher is not None:
             self.mclient.set_credentials_auto_refresher(credentials_refresher)
         self.mproject = project

aliyun-log-python-sdk-0.9.1/aliyun/log/getlogsresponse.py

@@ -0,0 +1,568 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+# Copyright (C) Alibaba Cloud Computing
+# All rights reserved.
+
+from .logresponse import LogResponse
+from .queriedlog import QueriedLog
+from .logexception import LogException
+from .util import Util
+from enum import Enum
+import json
+
+class GetLogsResponse(LogResponse):
+    """ The response of the GetLog API from log.
+
+    :type resp: dict
+    :param resp: GetLogsResponse HTTP response body
+
+    :type header: dict
+    :param header: GetLogsResponse HTTP response header
+    """
+
+    class QueryMode(Enum):
+        """ The enum of query type"""
+        NORMAL = 0
+        PHRASE = 1
+        SCAN = 2
+        SCAN_SQL = 3
+
+    def __init__(self, resp, header):
+        LogResponse.__init__(self, header, resp)
+        try:
+            self._meta = GetLogsResponse.GetLogsResponseMeta(
+                resp.get("meta"))
+            self._logs = []
+            for log in resp["data"]:
+                self._logs.append(QueriedLog._from_dict(log))
+
+        except Exception as ex:
+            raise LogException("InvalidResponse",
+                               "Failed to parse GetLogResponse, \nheader: "
+                               + str(header) + " \nBody:"
+                               + str(resp) + " \nOther: " + str(ex),
+                               resp_header=header,
+                               resp_body=resp)
+
+    @staticmethod
+    def _from_v1_resp(resp, header):
+        """
+        Support old api with best effort
+        """
+        meta_dict = {
+            'count': len(resp),
+            'progress': Util.h_v_t(header, 'x-log-progress'),
+            'processedRows': int(Util.h_v_td(header, 'x-log-processed-rows', '0')),
+            'elapsedMillisecond': int(Util.h_v_td(header, 'x-log-elapsed-millisecond', '0')),
+            'hasSQL': (Util.h_v_td(header, 'x-log-has-sql', 'false') != 'false'),
+            'whereQuery': Util.h_v_td(header, 'x-log-where-query', ''),
+            'aggQuery': Util.h_v_td(header, 'x-log-agg-query', ''),
+            'cpuSec': float(Util.h_v_td(header, 'x-log-cpu-sec', '0')),
+            'cpuCores': int(Util.h_v_td(header, 'x-log-cpu-cores', '0')),
+        }
+        # parse query info
+        query_info_str = Util.h_v_td(header, 'x-log-query-info', '')
+        query_info = json.loads(query_info_str)
+        meta_dict['mode'] = Util.h_v_td(query_info, 'mode', 0)
+        query_mode = GetLogsResponse.QueryMode(meta_dict['mode'])
+        if query_mode in (GetLogsResponse.QueryMode.SCAN, GetLogsResponse.QueryMode.SCAN_SQL):
+            meta_dict['scanBytes'] = Util.h_v_td(query_info, 'scanBytes', 0)
+        if query_mode in (GetLogsResponse.QueryMode.PHRASE, GetLogsResponse.QueryMode.SCAN):
+            scan_query_info = Util.h_v_td(query_info, 'phraseQueryInfo', dict())
+            meta_dict['phraseQueryInfo'] = {
+                'scanAll': (Util.h_v_td(scan_query_info, 'scanAll', 'false') != 'false'), 
+                'beginOffset': int(Util.h_v_td(scan_query_info, 'beginOffset', '0')),
+                'endOffset': int(Util.h_v_td(scan_query_info, 'endOffset', '0'))
+            }
+
+        return GetLogsResponse({
+            "meta": meta_dict,
+            "data": resp
+        }, header)
+    
+    def get_count(self):
+        """ Get log number from the response
+
+        :return: int, log number
+        """
+        return self.get_meta().get_count()
+
+    def is_completed(self):
+        """ Check if the get logs query is completed
+
+        :return: bool, true if this logs query is completed
+        """
+        return self.get_meta().is_completed()
+
+    def get_logs(self):
+        """ Get all logs from the response
+
+        :return: QueriedLog list, all log data
+        """
+        return self._logs
+
+    def get_processed_rows(self):
+        """ Get processed rows from the response
+
+        :return: int, processed_rows
+        """
+        return self.get_meta().get_processed_rows()
+
+    def get_elapsed_mills(self):
+        """ Get elapsed mills from the response
+
+        :return: int, elapsed_mills 
+        """
+        return self.get_meta().get_elapsed_millisecond()
+
+    def get_has_sql(self):
+        """ Get whether has sql from the response
+
+        :return: boolean, has_sql
+        """
+        return self.get_meta().get_has_sql()
+
+    def get_where_query(self):
+        """ Get the Search part of "Search|Analysis"
+
+        :return: str, where_query
+        """
+        return self.get_meta().get_where_query()
+
+    def get_agg_query(self):
+        """ Get the Analysis part of "Search|Analysis"
+
+        :return: str, agg_query
+        """
+        return self.get_meta().get_agg_query()
+
+    def get_cpu_sec(self):
+        """ Get cpu seconds used from the response
+
+        :return: float, cpu_sec
+        """
+        return self.get_meta().get_cpu_sec()
+
+    def get_cpu_cores(self):
+        """ Get cpu cores used from the response
+
+        :return: int, cpu_cores
+        """
+        return self.get_meta().get_cpu_cores()
+
+    def get_query_mode(self):
+        """ Get query_mode from the response
+
+        :return: GetLogsResponse.QueryMode, query_mode
+        """
+        
+        return GetLogsResponse.QueryMode(self.get_meta().get_mode())
+
+    def get_scan_bytes(self):
+        """ Get scan_bytes from the response
+
+        :return: int, scan_bytes
+        """
+        return self.get_meta().get_scan_bytes()
+
+    def get_begin_offset(self):
+        """ Get begin_offset from the response
+
+        :return: int, begin_offset
+        """
+        phrase_query_info = self.get_meta().get_phrase_query_info()
+        return phrase_query_info.get_begin_offset() if phrase_query_info else None
+
+    def get_end_offset(self):
+        """ Get end_offset from the response
+
+        :return: int, end_offset
+        """
+        phrase_query_info = self.get_meta().get_phrase_query_info()
+        return phrase_query_info.get_end_offset() if phrase_query_info else None
+
+    def get_scan_all(self):
+        """ Get scan_all from the response
+
+        :return: bool, scan_all
+        """
+        phrase_query_info = self.get_meta().get_phrase_query_info()
+        return phrase_query_info.get_scan_all() if phrase_query_info else None
+
+    def log_print(self):
+        print('GetLogsResponse:')
+        print('headers:', self.get_all_headers())
+        print('count:', self.get_count())
+        print('progress:', self.get_meta().get_progress())
+        print('meta:', self.get_meta()._to_dict())
+        print("\nQueriedLog class:\n")
+        for log in self.get_logs():
+            log.log_print()
+            print("\n")
+
+    def get_log_list(self):
+        print('count:', self.get_count())
+        print('progress:', self.get_meta().get_progress())
+        return self.get_logs()
+
+    def get_meta(self):
+        """ Get meta from the response
+
+        :return: meta, GetLogsResponseMeta
+        """
+        return self._meta
+
+    def merge(self, other):
+        """ merge with other response
+            merge logs and meta
+        """
+        if not isinstance(other, GetLogsResponse):
+            raise ValueError(
+                "passed response is not a GetLogsResponse: " + str(type(other)))
+        if other is None:
+            return self
+        self.get_meta().merge(other.get_meta())
+        self.get_logs().extend(other.get_logs())
+        return self
+
+    class GetLogsResponseMeta():
+        """ The meta info of get logs response
+
+        """
+
+        def __init__(self, meta):
+            self._count = Util.v_or_d(meta.get("count"), 0)
+            self._progress = meta.get("progress")
+            self._processed_rows = Util.v_or_d(meta.get("processedRows"), 0)
+            self._elapsed_millisecond = Util.v_or_d(meta.get("elapsedMillisecond"), 0)
+            self._has_sql = Util.v_or_d(meta.get("hasSQL"), False)
+            self._where_query = Util.v_or_d(meta.get("whereQuery"), '')
+            self._agg_query = Util.v_or_d(meta.get("aggQuery"))
+            self._cpu_sec = Util.v_or_d(meta.get("cpuSec"), 0.0)
+            self._cpu_cores = Util.v_or_d(meta.get("cpuCores"), 0)
+            self._mode = Util.v_or_d(meta.get("mode"), 0)
+            self._scan_bytes = Util.v_or_d(meta.get("scanBytes"), 0)
+            phrase_query_info = meta.get("phraseQueryInfo")
+            self._phrase_query_info = GetLogsResponse.PhraseQueryInfo(
+                phrase_query_info) if phrase_query_info else None
+
+            # self._limited = meta.get("limited")
+            # self._processed_bytes = meta.get("processedBytes")
+            # self._telemetry_type = meta.get("telementryType")  # not typo
+            # self._power_sql = Util.v_or_d(meta.get("powerSql"), False)
+            # self._inserted_sql = meta.get("insertedSQL")
+            # self._keys = meta.get("keys")
+            # self._marker = meta.get("marker")
+            # self._shard = meta.get("shard")
+            # self._is_accurate = meta.get("isAccurate")
+            # self._column_types = meta.get("columnTypes")
+            # self._highlights = meta.get("highlights")
+            # self._terms = []
+            # for term in meta.get("terms", []):
+            #     self._terms.append(GetLogsResponse.Term._from_dict(term))
+
+        def is_completed(self):
+            """ Check if the get logs query is completed
+
+            :return: bool, true if this logs query is completed
+            """
+            return self._progress == 'Complete'
+
+        def merge(self, other):
+            """ merge with other GetLogsResponseMeta
+
+            """
+            self._progress = other.progress
+            self._count += other.count
+            self._processed_rows += other.processed_rows
+            return self
+
+        def get_count(self):
+            """ Get log number from the response
+
+            :return: int, log number
+            """
+            return self._count
+
+        def _to_dict(self):
+            """ to Dict
+            """
+            phrase_query_info = None
+            if self._phrase_query_info is not None:
+                phrase_query_info = self._phrase_query_info._to_dict()
+
+            return {
+                'count': self._count,
+                'progress': self._progress,
+                'processedRows': self._processed_rows,
+                'elapsedMillisecond': self._elapsed_millisecond,
+                'hasSQL': self._has_sql,
+                'whereQuery': self._where_query,
+                'aggQuery': self._agg_query,
+                'cpuSec': self._cpu_sec,
+                'cpuCores': self._cpu_cores,
+                'mode': self._mode,
+                'scanBytes': self._scan_bytes,
+                'phraseQueryInfo': phrase_query_info,
+                # 'limited': self._limited,
+                # 'processedBytes': self._processed_bytes,
+                # 'telementryType': self._telemetry_type,  # not typo
+                # 'powerSql': self._power_sql,
+                # 'insertedSQL': self._inserted_sql,
+                # 'keys': self._keys,
+                # 'marker': self._marker,
+                # 'shard': self._shard,
+                # 'isAccurate': self._is_accurate,
+                # 'columnTypes': self._column_types,
+                # 'highlights': self._highlights,
+                # 'terms': [term.to_dict() for term in self._terms],
+            }
+
+        def log_print(self):
+            """ print infos
+            """
+            print(self._to_dict())
+
+        def get_progress(self):
+            """ Progress of get logs, returns 'Complete' if completed
+
+            :return: str
+            """
+            return self._progress
+
+        def get_processed_rows(self):
+            """ Get processed rows from the response
+
+            :return: processed_rows, int
+            """
+            return self._processed_rows
+
+        def get_elapsed_millisecond(self):
+            """ Get elapsed mills from the response
+
+            :return: elapsed_millisecond, int
+            """
+            return self._elapsed_millisecond
+
+        def get_has_sql(self):
+            """ Get whether has sql from the response
+
+            :return: has_sql, boolean
+            """
+            return self._has_sql
+
+        def get_where_query(self):
+            """ Get the Search part of "Search|Analysis"
+
+            :return: where_query, str
+            """
+            return self._where_query
+
+        def get_agg_query(self):
+            """ Get the Analysis part of "Search|Analysis"
+
+            :return: agg_query, str
+            """
+            return self._agg_query
+
+        def get_cpu_sec(self):
+            """ Get cpu seconds used from the response
+
+            :return: cpu_sec, float64
+            """
+            return self._cpu_sec
+
+        def get_cpu_cores(self):
+            """ Get cpu cores used from the response
+
+            :return: cpu_cores, float64
+            """
+            return self._cpu_cores
+
+        def get_mode(self):
+            """ Get query mode from the response
+
+            :return: mode
+            """
+            return self._mode
+
+        def get_scan_bytes(self):
+            """ Get scan_bytes from the response
+
+            :return: scan_bytes, int
+            """
+            return self._scan_bytes
+
+        def get_phrase_query_info(self):
+            """ 
+            :return: phraseQueryInfo, PhraseQueryInfo
+            """
+            return self._phrase_query_info
+
+        # def get_limited(self):
+        #     """ 
+        #     :return: limited, int
+        #     """
+        #     return self._limited
+
+        # def get_processed_bytes(self):
+        #     """ 
+        #     :return: processed_bytes, int
+        #     """
+        #     return self._processed_bytes
+
+        # def get_telemetry_type(self):
+        #     """ 
+        #     :return: telemetry_type, str
+        #     """
+        #     return self._telemetry_type
+
+        # def get_power_sql(self):
+        #     """ 
+        #     :return: power_sql, bool
+        #     """
+        #     return self._power_sql
+
+        # def get_inserted_sql(self):
+        #     """ 
+        #     :return: inserted_sql, str
+        #     """
+        #     return self._inserted_sql
+
+        # def get_keys(self):
+        #     """ 
+        #     :return: keys, List[str]
+        #     """
+        #     return self._keys
+
+        # def get_marker(self):
+        #     """ 
+        #     :return: marker, str
+        #     """
+        #     return self._marker
+
+        # def get_shard(self):
+        #     """ 
+        #     :return: shard, int
+        #     """
+        #     return self._shard
+
+        # def get_is_accurate(self):
+        #     """ 
+        #     :return: is_accurate, bool
+        #     """
+        #     return self._is_accurate
+
+        # def get_column_types(self):
+        #     """ 
+        #     :return: column_types, List[str]
+        #     """
+        #     return self._column_types
+
+        # def get_highlights(self):
+        #     """ 
+        #     :return: highlights, List[Dict]
+        #     """
+        #     return self._highlights
+        
+        # def get_terms(self):
+        #     """ 
+        #     :return: terms, List[Term]
+        #     """
+        #     return self._terms
+
+    class PhraseQueryInfo():
+        """ query info of phrase, includes beginOffset/endOffset/scanAll/endTime
+
+        """
+
+        def __init__(self, pharse_query_info):
+            self._begin_offset = pharse_query_info.get("beginOffset")
+            self._end_offset = pharse_query_info.get("endOffset")
+            self._scan_all = pharse_query_info.get("scanAll")
+            self._end_time = pharse_query_info.get("endTime")
+
+        def get_begin_offset(self):
+            """ Get begin_offset from the response
+
+            :return: begin_offset, int
+            """
+            return self._begin_offset
+
+        def get_end_offset(self):
+            """ Get end_offset from the response
+
+            :return: end_offset, int
+            """
+            return self._end_offset
+
+        def get_scan_all(self):
+            """ Get scan_all from the response
+
+            :return: scan_all, bool
+            """
+            return self._scan_all
+
+        def get_end_time(self):
+            """ Get end_time  from the response
+
+            :return: end_time, int
+            """
+            return self._end_time
+
+        def _to_dict(self):
+            """ to Dict
+            """
+            return {
+                "beginOffset": self._begin_offset,
+                "endOffset": self._end_offset,
+                "scanAll": self._scan_all,
+                "endTime": self._end_time
+            }
+
+        def log_print(self):
+            """ print info
+
+            """
+            print(self._to_dict())
+
+
+    # class Term():
+    #     """ terms of query, field key/term
+    #     """
+
+    #     def __init__(self, key, term):
+    #         self._key = key
+    #         self._term = term
+
+    #     @classmethod
+    #     def _from_dict(cls, data):
+    #         """ Initialize from a dict
+    #         """
+    #         key = data.get("key")
+    #         term = data.get("term")
+    #         return cls(key, term)
+
+    #     def get_key(self):
+    #         """ field key of term
+    #         """
+    #         return self._key
+
+    #     def get_term(self):
+    #         """ field term of term
+    #         """
+    #         return self._term
+
+    #     def _to_dict(self):
+    #         """ to Dict
+    #         """
+    #         return {
+    #             "key": self._key,
+    #             "term": self._term
+    #         }
+
+    #     def log_print(self):
+    #         """ print info
+    #         """
+    #         print(self._to_dict())

{aliyun-log-python-sdk-0.9.0 → aliyun-log-python-sdk-0.9.1}/aliyun/log/logclient.py

@@ -173,6 +173,7 @@ class LogClient(object):
         self._auth_version = auth_version
         self._region = region
         self._auth = make_auth(accessKeyId, accessKey, auth_version, region)
+        self._get_logs_v2_enabled = True
 
     def _replace_credentials(self):
         delta = time.time() - self._last_refresh
@@ -633,10 +634,8 @@ class LogClient(object):
             headers = {}
             params = {'from': parse_timestamp(from_time),
                       'to': parse_timestamp(to_time),
-                      'type': 'log',
                       'line': size,
                       'offset': offset,
-                      'reverse': 'true' if reverse else 'false',
                       'powerSql': power_sql,
                       'accurate': accurate_query,
                       'fromNs': from_time_nano_part,
@@ -650,10 +649,29 @@ class LogClient(object):
             if scan:
                 params['session'] = 'mode=scan'
                 params['forward'] = 'true' if forward else 'false'
-
-            resource = "/logstores/" + logstore
-            (resp, header) = self._send("GET", project, None, resource, params, headers)
-            ret = GetLogsResponse(resp, header)
+            
+            if self._get_logs_v2_enabled:
+                resource = "/logstores/" + logstore + "/logs"
+                headers["Content-Type"] = "application/json"
+                params['reverse'] = reverse
+                params['forward'] = forward
+                body_str = six.b(json.dumps(params))
+                headers["x-log-bodyrawsize"] = str(len(body_str))
+                accept_encoding = "lz4" if Util.is_lz4_available() else "deflate"
+                headers['Accept-Encoding'] = accept_encoding
+
+                (resp, header) = self._send("POST", project, body_str, resource, None, headers, respons_body_type=accept_encoding)
+
+                raw_data = Util.uncompress_response(header, resp)
+                exJson = self._loadJson(200, header, raw_data, requestId=Util.h_v_td(header, 'x-log-requestid', ''))
+                exJson = Util.convert_unicode_to_str(exJson)
+                ret = GetLogsResponse(exJson, header)
+            else:
+                resource = "/logstores/" + logstore
+                params['type'] = 'log'
+                params['reverse'] = 'true' if reverse else 'false'
+                (resp, header) = self._send("GET", project, None, resource, params, headers)
+                ret = GetLogsResponse._from_v1_resp(resp, header)
             if ret.is_completed():
                 break
 
@@ -804,7 +822,7 @@ class LogClient(object):
             if query_mode is GetLogsResponse.QueryMode.NORMAL or query_mode is GetLogsResponse.QueryMode.SCAN_SQL:
                 offset += count
             else:
-                scan_all = (response.get_scan_all() == 'true')
+                scan_all = response.get_scan_all()
                 if forward:
                     offset = response.get_end_offset()
                 else:

{aliyun-log-python-sdk-0.9.0 → aliyun-log-python-sdk-0.9.1}/aliyun/log/queriedlog.py

@@ -4,7 +4,7 @@
 # Copyright (C) Alibaba Cloud Computing
 # All rights reserved.
 
-
+import six
 class QueriedLog(object):
     """ The QueriedLog is a log of the GetLogsResponse which obtained from the log.
 
@@ -23,6 +23,27 @@ class QueriedLog(object):
         self.source = source
         self.contents = contents
 
+    @staticmethod
+    def _from_dict(data):
+        """ Initalize from dict
+        """
+        source = data.get("__source__", "")
+        time = data.get("__time__", '0')
+        contents = {}
+        for key in six.iterkeys(data):
+            if key not in ['__time__', '__source__']:
+                contents[key] = data[key]
+
+        return QueriedLog(time, source, contents)
+
+    def _to_dict(self):
+        res = {
+            '__time__': str(self.timestamp),
+            '__source__': self.source,
+        }
+        res.update(self.contents)
+        return res
+
     def get_time(self):
         """ Get log time