aliyun-log-python-sdk 0.9.5__tar.gz → 0.9.7__tar.gz

{aliyun-log-python-sdk-0.9.5/aliyun_log_python_sdk.egg-info → aliyun-log-python-sdk-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.5
+Version: 0.9.7
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
@@ -16,13 +16,6 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 License-File: LICENSE
-Requires-Dist: six
-Requires-Dist: requests
-Requires-Dist: python-dateutil
-Requires-Dist: elasticsearch
-Requires-Dist: jmespath
-Requires-Dist: dateparser
-Requires-Dist: protobuf<4.0.0,>3.4.0
 
 
 Python SDK for Alicloud Log Service

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/es_migration/mapping_index_converter.py

@@ -34,6 +34,7 @@ class MappingIndexConverter(object):
         line_config = IndexLineConfig(token_list=cls.DEFAULT_TOKEN_LIST, chinese=True)
         key_config_list = cls.to_key_config_list(properties=mapping["properties"])
         index_config = IndexConfig(line_config=line_config, key_config_list=key_config_list)
+        index_config.docvalue_max_text_len = 16384
         return index_config
 
     @classmethod
@@ -43,6 +44,8 @@ class MappingIndexConverter(object):
             return key_config_list
         fields = set()
         for field_name, field_desc in properties.items():
+            if len(fields) >= 16:
+                break
             if field_name.lower() in fields:
                 continue
             fields.add(field_name.lower())
@@ -250,11 +253,12 @@ class MappingIndexConverter(object):
         key_configs = cls.parse_properties(None, properties)
         for key_name, key_config in key_configs.items():
             json_key_config.add_key(key_name=key_name, key_type=key_config.index_type, doc_value=key_config.doc_value)
-            if len(json_key_config.json_keys) >= 50:
+            if len(json_key_config.json_keys) >= 8:
                 break
         return IndexKeyConfig(
             index_type=AliyunLogFieldType.JSON,
-            json_key_config=json_key_config
+            json_key_config=json_key_config,
+            doc_value=True,
         )
 
     @classmethod

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/es_migration/migration_task.py

@@ -11,7 +11,7 @@ import json
 import traceback
 from datetime import datetime
 from elasticsearch.exceptions import NotFoundError
-from aliyun.log import LogClient
+from aliyun.log import LogClient, LogException
 from aliyun.log.putlogsrequest import PutLogsRequest
 from aliyun.log.es_migration.doc_logitem_converter import DocLogItemConverter
 
@@ -251,12 +251,29 @@ class MigrationLogstore(object):
         return self._logstore_name
 
     def put_logs(self, logitems):
-        self._log_client.put_logs(
-            PutLogsRequest(
-                project=self._project_name,
-                logstore=self._logstore_name,
-                topic=self._topic,
-                source=self._source,
-                logitems=logitems,
+        try:
+            self._log_client.put_logs(
+                PutLogsRequest(
+                    project=self._project_name,
+                    logstore=self._logstore_name,
+                    topic=self._topic,
+                    source=self._source,
+                    logitems=logitems,
+                )
+            )
+        except LogException as exc:
+            if exc.get_error_code() != "InvalidLogSize":
+                raise
+        else:  # putting succeeds
+            return
+
+        for item in logitems:
+            self._log_client.put_logs(
+                PutLogsRequest(
+                    project=self._project_name,
+                    logstore=self._logstore_name,
+                    topic=self._topic,
+                    source=self._source,
+                    logitems=[item],
+                )
             )
-        )

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/export_response.py

@@ -8,7 +8,7 @@
 from .logresponse import LogResponse
 from .util import Util
 
-__all__ = ['CreateExportResponse', 'DeleteExportResponse', 'GetExportResponse', 'ListExportResponse']
+__all__ = ['CreateExportResponse', 'DeleteExportResponse', 'GetExportResponse', 'ListExportResponse', 'UpdateExportResponse']
 
 
 class CreateExportResponse(LogResponse):
@@ -32,6 +32,15 @@ class DeleteExportResponse(LogResponse):
         print('headers:', self.get_all_headers())
 
 
+class UpdateExportResponse(LogResponse):
+    def __init__(self, header, resp=''):
+        LogResponse.__init__(self, header, resp)
+
+    def log_print(self):
+        print('UpdateExportResponse:')
+        print('headers:', self.get_all_headers())
+
+
 class GetExportResponse(LogResponse):
     """ The response of the get_export API from log.
 

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/getlogsresponse.py

@@ -248,20 +248,19 @@ class GetLogsResponse(LogResponse):
             self._phrase_query_info = GetLogsResponse.PhraseQueryInfo(
                 phrase_query_info) if phrase_query_info else None
 
-            # self._limited = meta.get("limited")
-            # self._processed_bytes = meta.get("processedBytes")
-            # self._telemetry_type = meta.get("telementryType")  # not typo
-            # self._power_sql = Util.v_or_d(meta.get("powerSql"), False)
-            # self._inserted_sql = meta.get("insertedSQL")
-            # self._keys = meta.get("keys")
-            # self._marker = meta.get("marker")
-            # self._shard = meta.get("shard")
-            # self._is_accurate = meta.get("isAccurate")
-            # self._column_types = meta.get("columnTypes")
-            # self._highlights = meta.get("highlights")
-            # self._terms = []
-            # for term in meta.get("terms", []):
-            #     self._terms.append(GetLogsResponse.Term._from_dict(term))
+            self._limited = meta.get("limited")
+            self._processed_bytes = meta.get("processedBytes")
+            self._telemetry_type = meta.get("telementryType") or meta.get("telemetryType") # not typo
+            self._power_sql = Util.v_or_d(meta.get("powerSql"), False)
+            self._inserted_sql = meta.get("insertedSQL")
+            self._keys = meta.get("keys")
+            self._marker = meta.get("marker")
+            self._is_accurate = meta.get("isAccurate")
+            self._column_types = meta.get("columnTypes")
+            self._highlights = meta.get("highlights")
+            self._terms = []
+            for term in meta.get("terms", []):
+                self._terms.append(GetLogsResponse.Term._from_dict(term))
 
         def is_completed(self):
             """ Check if the get logs query is completed
@@ -305,18 +304,17 @@ class GetLogsResponse(LogResponse):
                 'mode': self.get_mode(),
                 'scanBytes': self.get_scan_bytes(),
                 'phraseQueryInfo': phrase_query_info_dict,
-                # 'limited': self._limited,
-                # 'processedBytes': self._processed_bytes,
-                # 'telementryType': self._telemetry_type,  # not typo
-                # 'powerSql': self._power_sql,
-                # 'insertedSQL': self._inserted_sql,
-                # 'keys': self._keys,
-                # 'marker': self._marker,
-                # 'shard': self._shard,
-                # 'isAccurate': self._is_accurate,
-                # 'columnTypes': self._column_types,
-                # 'highlights': self._highlights,
-                # 'terms': [term.to_dict() for term in self._terms],
+                'limited': self.get_limited(),
+                'processedBytes': self.get_processed_bytes(),
+                'telementryType': self.get_telemetry_type(),  # not typo
+                'powerSql': self.get_power_sql(),
+                'insertedSQL': self.get_inserted_sql(),
+                'keys': self.get_keys(),
+                'marker': self.get_marker(),
+                'isAccurate': self.get_is_accurate(),
+                'columnTypes': self.get_column_types(),
+                'highlights': self.get_highlights(),
+                'terms': [term._to_dict() for term in self.get_terms()],
             }
 
         def log_print(self):
@@ -400,77 +398,72 @@ class GetLogsResponse(LogResponse):
             """
             return self._phrase_query_info
 
-        # def get_limited(self):
-        #     """ 
-        #     :return: limited, int
-        #     """
-        #     return self._limited
-
-        # def get_processed_bytes(self):
-        #     """ 
-        #     :return: processed_bytes, int
-        #     """
-        #     return self._processed_bytes
-
-        # def get_telemetry_type(self):
-        #     """ 
-        #     :return: telemetry_type, str
-        #     """
-        #     return self._telemetry_type
-
-        # def get_power_sql(self):
-        #     """ 
-        #     :return: power_sql, bool
-        #     """
-        #     return self._power_sql
-
-        # def get_inserted_sql(self):
-        #     """ 
-        #     :return: inserted_sql, str
-        #     """
-        #     return self._inserted_sql
-
-        # def get_keys(self):
-        #     """ 
-        #     :return: keys, List[str]
-        #     """
-        #     return self._keys
-
-        # def get_marker(self):
-        #     """ 
-        #     :return: marker, str
-        #     """
-        #     return self._marker
-
-        # def get_shard(self):
-        #     """ 
-        #     :return: shard, int
-        #     """
-        #     return self._shard
-
-        # def get_is_accurate(self):
-        #     """ 
-        #     :return: is_accurate, bool
-        #     """
-        #     return self._is_accurate
-
-        # def get_column_types(self):
-        #     """ 
-        #     :return: column_types, List[str]
-        #     """
-        #     return self._column_types
-
-        # def get_highlights(self):
-        #     """ 
-        #     :return: highlights, List[Dict]
-        #     """
-        #     return self._highlights
+        def get_limited(self):
+            """ 
+            :return: limited, int
+            """
+            return self._limited
+
+        def get_processed_bytes(self):
+            """ 
+            :return: processed_bytes, int
+            """
+            return self._processed_bytes
+
+        def get_telemetry_type(self):
+            """ 
+            :return: telemetry_type, str
+            """
+            return self._telemetry_type
+
+        def get_power_sql(self):
+            """ 
+            :return: power_sql, bool
+            """
+            return self._power_sql
+
+        def get_inserted_sql(self):
+            """ 
+            :return: inserted_sql, str
+            """
+            return self._inserted_sql
+
+        def get_keys(self):
+            """ 
+            :return: keys, List[str]
+            """
+            return self._keys
+
+        def get_marker(self):
+            """ 
+            :return: marker, str
+            """
+            return self._marker
+
+
+        def get_is_accurate(self):
+            """ 
+            :return: is_accurate, bool
+            """
+            return self._is_accurate
+
+        def get_column_types(self):
+            """ 
+            :return: column_types, List[str]
+            """
+            return self._column_types
+
+        def get_highlights(self):
+            """ 
+            :return: highlights, List[Dict]
+            """
+            return self._highlights
         
-        # def get_terms(self):
-        #     """ 
-        #     :return: terms, List[Term]
-        #     """
-        #     return self._terms
+        def get_terms(self):
+            """ 
+            :return: terms, List[Term]
+            """
+            return self._terms
 
     class PhraseQueryInfo():
         """ query info of phrase, includes beginOffset/endOffset/scanAll/endTime
@@ -528,41 +521,41 @@ class GetLogsResponse(LogResponse):
             print(self._to_dict())
 
 
-    # class Term():
-    #     """ terms of query, field key/term
-    #     """
-
-    #     def __init__(self, key, term):
-    #         self._key = key
-    #         self._term = term
-
-    #     @classmethod
-    #     def _from_dict(cls, data):
-    #         """ Initialize from a dict
-    #         """
-    #         key = data.get("key")
-    #         term = data.get("term")
-    #         return cls(key, term)
-
-    #     def get_key(self):
-    #         """ field key of term
-    #         """
-    #         return self._key
-
-    #     def get_term(self):
-    #         """ field term of term
-    #         """
-    #         return self._term
-
-    #     def _to_dict(self):
-    #         """ to Dict
-    #         """
-    #         return {
-    #             "key": self._key,
-    #             "term": self._term
-    #         }
-
-    #     def log_print(self):
-    #         """ print info
-    #         """
-    #         print(self._to_dict())
+    class Term():
+        """ terms of query, field key/term
+        """
+
+        def __init__(self, key, term):
+            self._key = key
+            self._term = term
+
+        @classmethod
+        def _from_dict(cls, data):
+            """ Initialize from a dict
+            """
+            key = data.get("key")
+            term = data.get("term")
+            return cls(key, term)
+
+        def get_key(self):
+            """ field key of term
+            """
+            return self._key
+
+        def get_term(self):
+            """ field term of term
+            """
+            return self._term
+
+        def _to_dict(self):
+            """ to Dict
+            """
+            return {
+                "key": self.get_key(),
+                "term": self.get_term()
+            }
+
+        def log_print(self):
+            """ print info
+            """
+            print(self._to_dict())

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/index_config.py

@@ -25,6 +25,7 @@ class IndexJsonKeyConfig(object):
         self.max_depth = max_depth
         self.alias = alias
         self.json_keys = {}
+        self.keys_set = set()
 
     """
     Inner key config in json, if the json value is :
@@ -52,6 +53,10 @@ class IndexJsonKeyConfig(object):
     def add_key(self, key_name, key_type, doc_value=False, alias=None):
         if key_type != 'text' and key_type != 'long' and key_type != 'double':
             return
+        if key_name.lower() in self.keys_set:
+            return
+        self.keys_set.add(key_name.lower())
+
         self.json_keys[key_name] = {}
         self.json_keys[key_name]["type"] = key_type
         self.json_keys[key_name]["doc_value"] = doc_value

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/logclient.py

@@ -5075,7 +5075,7 @@ class LogClient(object):
         :type project_name: string
         :param project_name: the Project name
 
-        :type export: string
+        :type export: Export
         :param export: the export job configuration
         """
         params = {}
@@ -5110,6 +5110,28 @@ class LogClient(object):
         (resp, header) = self._send("DELETE", project_name, None, resource, params, headers)
         return DeleteExportResponse(header, resp)
 
+    def update_export(self, project_name, job_name, export):
+        """ Update and Restart an export job
+        Unsuccessful opertaion will cause an LogException.
+
+        :type project_name: string
+        :param project_name: the Project name
+
+        :type job_name: string
+        :param job_name: the job name of export job
+
+        :type export: string
+        :param export: the export job configuration
+        """
+        if not isinstance(export, str):
+            raise TypeError("export type must be string")
+        params = {"action": "RESTART"}
+        body = six.b(export)
+        headers = {'Content-Type': 'application/json', 'x-log-bodyrawsize': str(len(body))}
+        resource = "/jobs/" + job_name
+        (resp, header) = self._send("PUT", project_name, body, resource, params, headers)
+        return UpdateExportResponse(header, resp)
+
     def get_export(self, project_name, job_name):
         """ get export
         Unsuccessful operation will cause an LogException.

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7}/aliyun/log/version.py

@@ -1,4 +1,4 @@
-__version__ = '0.9.5'
+__version__ = '0.9.7'
 
 import sys
 OS_VERSION = str(sys.platform)

{aliyun-log-python-sdk-0.9.5 → aliyun-log-python-sdk-0.9.7/aliyun_log_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aliyun-log-python-sdk
-Version: 0.9.5
+Version: 0.9.7
 Summary: Aliyun log service Python client SDK
 Home-page: https://github.com/aliyun/aliyun-log-python-sdk
 Author: Aliyun
@@ -16,13 +16,6 @@ Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 License-File: LICENSE
-Requires-Dist: six
-Requires-Dist: requests
-Requires-Dist: python-dateutil
-Requires-Dist: elasticsearch
-Requires-Dist: jmespath
-Requires-Dist: dateparser
-Requires-Dist: protobuf<4.0.0,>3.4.0
 
 
 Python SDK for Alicloud Log Service