huaweicloud-encryption-sdk-python 1.2.1__py3-none-any.whl

huaweicloud_encryptionsdk/_lts_sdk/producer/core/mover.py

@@ -0,0 +1,69 @@
+import threading
+import time
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.retry_queue import RetryQueue
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class Mover(threading.Thread):
+    def __init__(self, io_worker, accumulator, io_thread_pool, config):
+        super().__init__()
+        self._io_worker = io_worker
+        self._accumulator = accumulator
+        self._io_thread_pool = io_thread_pool
+        self._retry_queue = RetryQueue()
+        self._config = config
+
+    def run(self):
+        while True:
+            sleep_ms = self._config.linger_ms
+            now_time_ms = common.current_time_ms()
+            self._accumulator.accumulator_lock.acquire_write()
+            try:
+                if self._accumulator.log_batch_map is None:
+                    map_count = 0
+                else:
+                    map_count = len(self._accumulator.log_batch_map)
+                for key, batch in self._accumulator.log_batch_map.items():
+                    time_interval = batch.create_time_ms + self._config.linger_ms - now_time_ms
+                    if time_interval <= 0:
+                        self._mover_send_to_server(key, batch, self._config)
+                    else:
+                        if sleep_ms > time_interval:
+                            sleep_ms = time_interval
+            finally:
+                self._accumulator.accumulator_lock.release_write()
+
+            if map_count == 0:
+                sleep_ms = self._config.linger_ms
+
+            retry_batch_list = self._retry_queue.get_retry_batch()
+            if retry_batch_list is None or len(retry_batch_list) == 0:
+                time.sleep(sleep_ms / 1000)
+            else:
+                for retry_batch in retry_batch_list:
+                    self._io_thread_pool.add_task(retry_batch)
+
+            self._accumulator.accumulator_lock.acquire_write()
+            try:
+                if self._accumulator.log_batch_map is not None:
+                    for key, batch in self._accumulator.log_batch_map.items():
+                        self._io_thread_pool.add_task(batch)
+
+                self._accumulator.log_batch_map = {}
+
+            finally:
+                self._accumulator.accumulator_lock.release_write()
+
+            batch_list = self._retry_queue.get_retry_batch()
+            if batch_list is not None:
+                for batch in batch_list:
+                    self._io_thread_pool.add_task(batch)
+
+    def _mover_send_to_server(self, key, batch, config):
+        if key not in self._accumulator.log_batch_map:
+            return
+        if common.current_time_ms() - self._accumulator.log_batch_map.get(key).create_time_ms < config.linger_ms:
+            return
+        self._io_thread_pool.add_task(batch)
+        self._accumulator.log_batch_map.pop(key)

huaweicloud_encryptionsdk/_lts_sdk/producer/core/producer.py

@@ -0,0 +1,98 @@
+from huaweicloud_encryptionsdk._lts_sdk.producer.callback.call_back import CallBack
+from huaweicloud_encryptionsdk._lts_sdk.producer.core.accumulator import Accumulator
+from huaweicloud_encryptionsdk._lts_sdk.producer.core.io_thread_pool import IOThreadPool
+from huaweicloud_encryptionsdk._lts_sdk.producer.core.io_worker import IOWorker
+from huaweicloud_encryptionsdk._lts_sdk.producer.core.mover import Mover
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import error
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.config import Config
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.log import Log
+
+
+class Producer(object):
+
+    def __init__(self, config):
+        if config is None:
+            config = Config()
+        self._producer_config = config.validate_config()
+        self._io_worker = None
+        self._accumulator = None
+        self._io_thread_pool = None
+        self._mover = None
+        self._buckets = config.bucket
+
+    @staticmethod
+    def init_producer(producer_config):
+        producer = Producer(producer_config)
+
+        io_worker = IOWorker(producer_config.max_io_workers, producer_config.no_retry_code_list, producer)
+        io_thread_pool = IOThreadPool(io_worker)
+        accumulator = Accumulator.init_accumulator(producer_config, io_worker, producer, io_thread_pool)
+        mover = Mover(io_worker, accumulator, io_thread_pool, producer_config)
+        producer.io_thread_pool = io_thread_pool
+        producer.accumulator = accumulator
+        producer.io_worker = io_worker
+        producer.mover = mover
+        return producer
+
+    @property
+    def producer_config(self):
+        return self._producer_config
+
+    @producer_config.setter
+    def producer_config(self, value):
+        self._producer_config = value
+
+    @property
+    def io_worker(self):
+        return self._io_worker
+
+    @io_worker.setter
+    def io_worker(self, value):
+        self._io_worker = value
+
+    @property
+    def buckets(self):
+        return self._buckets
+
+    @buckets.setter
+    def buckets(self, value):
+        self._buckets = value
+
+    @property
+    def accumulator(self):
+        return self._accumulator
+
+    @accumulator.setter
+    def accumulator(self, value):
+        self._accumulator = value
+
+    @property
+    def io_thread_pool(self):
+        return self._io_thread_pool
+
+    @io_thread_pool.setter
+    def io_thread_pool(self, value):
+        self._io_thread_pool = value
+
+    @property
+    def mover(self):
+        return self._mover
+
+    @mover.setter
+    def mover(self, value):
+        self._mover = value
+
+    def start_producer(self):
+        self._mover.name = "lts-mover-thread"
+        self._mover.start()
+        self._io_thread_pool.name = "lts-io-thread-pool-thread"
+        self._io_thread_pool.start()
+
+    def send_log(self, group_id: str, stream_id: str, log: Log):
+        return self._accumulator.add_log_to_log_batch_map(log, group_id, stream_id, None)
+
+    def send_log_with_callback(self, group_id: str, stream_id: str, log: Log, call_back: CallBack):
+        if call_back is None:
+            return error.Error(error_code=error.PARAMETER_ERROR_CODE,
+                               error_msg="call back fn is None, please check")
+        return self._accumulator.add_log_to_log_batch_map(log, group_id, stream_id, call_back)

huaweicloud_encryptionsdk/_lts_sdk/producer/lts_http/http_client.py

@@ -0,0 +1,63 @@
+import requests
+import urllib3
+from loguru import logger
+from requests.adapters import HTTPAdapter
+from urllib3.exceptions import InsecureRequestWarning
+from urllib3.util.retry import Retry
+
+from huaweicloud_encryptionsdk._lts_sdk.auth.signer import Signer
+from huaweicloud_encryptionsdk._lts_sdk.producer.lts_http.simple_request import LtsRequest
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import error
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.config import Config
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.log import LogGroup
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class HttpClient(object):
+    def __init__(self):
+        self._session = requests.Session()
+        self._signer = Signer()
+
+        retry_strategy = Retry(
+            total=2,
+            status_forcelist=[429, 500, 502, 503, 504],
+            backoff_factor=1
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy, pool_connections=100, pool_maxsize=100)
+        self._session.mount("https://", adapter)
+        self._session.mount("http://", adapter)
+        urllib3.disable_warnings(InsecureRequestWarning)
+
+    def put_logs(self, group_id, stream_id, log_group: LogGroup, config: Config):
+        if log_group is None or len(log_group.log_items) == 0:
+            return None
+
+        log_num = 0
+        for item in log_group.log_items:
+            log_num += len(item.contents)
+
+        body = common.convert_obj2_json_str(log_group)
+        header = {"content-type": "application/json"}
+
+        interface = "/v2/internal/{0}/lts/groups/{1}/streams/{2}/tenant/batch-contents".format(config.project_id,
+                                                                                               group_id, stream_id)
+
+        req = LtsRequest(method="POST", host=config.endpoint, headers=header, body=body, region=config.region_id,
+                         interface=interface, ak=config.access_key, sk=config.access_secret,
+                         security_token=config.security_token)
+        request = self._signer.sign(req)
+        r = request.prepare()
+        try:
+            logger.info("start send log num is {}", log_num)
+            response = self._session.send(r, verify=False)
+            response.encoding = "utf-8"
+            logger.info("send log status code {}, response is {}, send log num {}", response.status_code,
+                        response.content.decode("utf-8"), log_num)
+            if response.status_code == 200:
+                return None
+            else:
+                return error.Error(error_code=error.HTTP_QUEST_ERROR_CODE, error_msg=response.content.decode("utf-8"),
+                                   http_code=response.status_code)
+        except Exception as e:
+            logger.error("send log error {}, log num {}", e, log_num)
+            return error.Error(error_code=error.HTTP_QUEST_ERROR_CODE, error_msg=str(e), http_code=0)

huaweicloud_encryptionsdk/_lts_sdk/producer/lts_http/simple_request.py

@@ -0,0 +1,14 @@
+
+class LtsRequest(object):
+
+    def __init__(self, method, host, headers, body, region, interface, ak, sk, security_token="", **kwargs):
+        self.body = body
+        self.method = method
+        self.headers = headers
+        self.host = host
+        self.query = kwargs
+        self.region = region
+        self.interface = interface
+        self.ak = ak
+        self.sk = sk
+        self.security_token = security_token

huaweicloud_encryptionsdk/_lts_sdk/producer/model/config.py

@@ -0,0 +1,50 @@
+class Config(object):
+
+    def __init__(self, max_block_sec=0, max_io_workers=200, batch_size_threshold_in_bytes=512 * 1024,
+                 batch_count_threshold=4096, linger_ms=2000, retries=3, base_retry_backoff_ms=100,
+                 max_retry_backoff_ms=5000, http_max_timeout_sec=60, bucket=64,
+                 no_retry_code_list=None, total_size_in_bytes=100 * 1024 * 1024,
+                 max_reserved_attempts=11):
+        if no_retry_code_list is None:
+            no_retry_code_list = {413, 429}
+        self.max_block_sec = max_block_sec
+        self.max_io_workers = max_io_workers
+        self.batch_size_threshold_in_bytes = batch_size_threshold_in_bytes
+        self.batch_count_threshold = batch_count_threshold
+        self.linger_ms = linger_ms
+        self.retries = retries
+        self.base_retry_backoff_ms = base_retry_backoff_ms
+        self.max_retry_backoff_ms = max_retry_backoff_ms
+        self.http_max_timeout_sec = http_max_timeout_sec
+        self.bucket = bucket
+        self.no_retry_code_list = no_retry_code_list
+        self.total_size_in_bytes = total_size_in_bytes
+        self.max_reserved_attempts = max_reserved_attempts
+
+        self.endpoint = ""
+        self.access_key = ""
+        self.access_secret = ""
+        self.security_token = ""
+        self.project_id = ""
+        self.region_id = ""
+
+    def validate_config(self):
+        if self.max_reserved_attempts <= 0:
+            self.max_reserved_attempts = 11
+        if self.batch_count_threshold > 40960 or self.batch_count_threshold <= 0:
+            self.batch_count_threshold = 40960
+        if self.batch_size_threshold_in_bytes > 512 * 1024 or self.batch_size_threshold_in_bytes <= 0:
+            self.batch_size_threshold_in_bytes = 512 * 1024
+
+        if self.max_io_workers <= 0:
+            self.max_io_workers = 10
+
+        if self.base_retry_backoff_ms <= 0:
+            self.base_retry_backoff_ms = 100
+
+        if self.total_size_in_bytes <= 0:
+            self.total_size_in_bytes = 100 * 1024 * 1024
+
+        if self.linger_ms < 100:
+            self.linger_ms = 2000
+        return self

huaweicloud_encryptionsdk/_lts_sdk/producer/model/error.py

@@ -0,0 +1,18 @@
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+PARAMETER_ERROR_CODE = "0000001"
+ADD_LOG_TO_BATCH_ERROR_CODE = "0000002"
+ADD_LOG_WAIT_TIMEOUT_ERROR_CODE = "0000003"
+HTTP_QUEST_ERROR_CODE = "0000004"
+RESULT_JSON_FORMAT_ERROR_CODE = "0000005"
+
+
+class Error(object):
+    def __init__(self, error_code: str, error_msg: str, http_code: int = 0, request_id: str = ""):
+        self.http_code = http_code
+        self.error_code = error_code
+        self.error_msg = error_msg
+        self.request_id = request_id
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self)

huaweicloud_encryptionsdk/_lts_sdk/producer/model/log.py

@@ -0,0 +1,87 @@
+from typing import List
+
+from huaweicloud_encryptionsdk._lts_sdk.utils.common import solv_log
+
+
+class LogContent(object):
+    def __init__(self, log_time_ns: int, log: str):
+        self.log_time_ns = log_time_ns
+        self.log = log
+
+    def getSize(self) -> int:
+        log_length = 0
+        if self.log:
+            log_length += len(self.log)
+
+        return log_length + solv_log(self.log_time_ns)
+
+    def __len__(self) -> int:
+        log_length = 0
+        if self.log:
+            log_length += len(self.log)
+
+        return log_length + solv_log(self.log_time_ns)
+
+
+class Log(object):
+    def __init__(self, contents: List[LogContent], labels: str):
+        self.contents = contents
+        self.labels = labels
+
+    def getSize(self) -> int:
+        log_length = 0
+        l = 0
+        if self.contents is None or len(self.contents) == 0:
+            return log_length
+
+        for content in self.contents:
+            if content is None:
+                l = 0
+            else:
+                l = content.getSize()
+
+            log_length += 1 + l + solv_log(l)
+        return log_length
+
+    def __len__(self) -> int:
+        log_length = 0
+        l = 0
+        if self.contents is None or len(self.contents) == 0:
+            return log_length
+
+        for content in self.contents:
+            if content is None:
+                l = 0
+            else:
+                l = len(content)
+
+            log_length += 1 + l + solv_log(l)
+        return log_length
+
+
+class LogGroup(object):
+    def __init__(self, log_items: List[Log]):
+        self.log_items = log_items
+
+    def is_empty_logs(self):
+        return self.log_items is None or len(self.log_items) == 0
+
+    def getSize(self) -> int:
+        log_length = 0
+        l = 0
+
+        if not self.is_empty_logs():
+            for log_item in self.log_items:
+                l = log_item.getSize()
+                log_length += 1 + l + solv_log(l)
+        return log_length
+
+    def __len__(self) -> int:
+        log_length = 0
+        l = 0
+
+        if not self.is_empty_logs():
+            for log_item in self.log_items:
+                l = len(log_item)
+                log_length += 1 + l + solv_log(l)
+        return log_length

huaweicloud_encryptionsdk/_lts_sdk/producer/model/log_batch.py

@@ -0,0 +1,154 @@
+import threading
+from typing import List
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.config import Config
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.log import LogGroup, Log
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.result import Result
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class LogBatch(object):
+    def __init__(self):
+        self._log_group = None
+        self._total_data_size = 0
+        self._log_group_count = 0
+        self._attempt_count = 0
+        self.lock = threading.Lock()
+        self._call_back_list = []
+        self._group_id = None
+        self._stream_id = None
+        self._config = None
+        self._create_time_ms = common.current_time_ms()
+        self._result = Result()
+        self._next_retry_ms = 0
+
+    @property
+    def result(self):
+        return self._result
+
+    @result.setter
+    def result(self, value):
+        self._result = value
+
+    @property
+    def next_retry_ms(self):
+        return self._next_retry_ms
+
+    @next_retry_ms.setter
+    def next_retry_ms(self, value):
+        self._next_retry_ms = value
+
+    @property
+    def create_time_ms(self) -> int:
+        return self._create_time_ms
+
+    @create_time_ms.setter
+    def create_time_ms(self, value: int):
+        self._create_time_ms = value
+
+    @property
+    def total_data_size(self) -> int:
+        return self._total_data_size
+
+    @total_data_size.setter
+    def total_data_size(self, value: int):
+        self._total_data_size = value
+
+    @property
+    def log_group_count(self) -> int:
+        return self._log_group_count
+
+    @log_group_count.setter
+    def log_group_count(self, value: int):
+        self._log_group_count = value
+
+    @property
+    def attempt_count(self) -> int:
+        return self._attempt_count
+
+    @attempt_count.setter
+    def attempt_count(self, value: int):
+        self._attempt_count = value
+
+    @property
+    def log_group(self) -> LogGroup:
+        return self._log_group
+
+    @log_group.setter
+    def log_group(self, value: LogGroup):
+        self._log_group = value
+
+    @property
+    def group_id(self) -> str:
+        return self._group_id
+
+    @group_id.setter
+    def group_id(self, value: str):
+        self._group_id = value
+
+    @property
+    def stream_id(self) -> str:
+        return self._stream_id
+
+    @stream_id.setter
+    def stream_id(self, value: str):
+        self._stream_id = value
+
+    @property
+    def config(self) -> Config:
+        return self._config
+
+    @config.setter
+    def config(self, value: Config):
+        self._config = value
+
+    @property
+    def call_back_list(self):
+        return self._call_back_list
+
+    @call_back_list.setter
+    def call_back_list(self, value):
+        self._call_back_list = value
+
+    def add_attempt(self, attempt):
+        if attempt is None:
+            return
+        if self._result is None:
+            self._result = Result()
+        if self._result.attempt_list is None:
+            self._result.attempt_list = []
+        self._result.attempt_list.append(attempt)
+
+    def add_call_back(self, call_back):
+        if call_back is None:
+            return
+        else:
+            self._call_back_list.append(call_back)
+
+    def add_log_to_log_group(self, log: Log):
+        self.log_group.log_items.append(log)
+
+    def add_total_data_size(self, delta):
+        self._total_data_size += delta
+
+
+def init_product_batch_var_single_log(log_data: Log, call_back, group_id: str, stream_id: str, config: Config):
+    logs = [log_data]
+    log_group = LogGroup(log_items=logs)
+    return _producer_batch(call_back, group_id, stream_id, config, log_group)
+
+
+def init_product_batch_var_multi_log(log_data: List[Log], call_back, group_id: str, stream_id: str, config: Config):
+    log_group = LogGroup(log_items=log_data)
+    return _producer_batch(call_back, group_id, stream_id, config, log_group)
+
+
+def _producer_batch(call_back, group_id: str, stream_id: str, config: Config, log_group: LogGroup):
+    log_batch = LogBatch()
+    log_batch.log_group = log_group
+    log_batch.add_call_back(call_back)
+    log_batch._group_id = group_id
+    log_batch.stream_id = stream_id
+    log_batch.config = config
+    log_batch.total_data_size = log_group.getSize()
+    return log_batch

huaweicloud_encryptionsdk/_lts_sdk/producer/model/lts_store.py

@@ -0,0 +1,21 @@
+import json
+import time
+from typing import List
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.log import Log, LogContent
+
+
+def generate_log(log_list: List[str], labels: dict) -> Log:
+    """
+
+    :rtype: object
+    """
+    log_contents = []
+    for log_str in log_list:
+        log_content = LogContent(time.time_ns(), log_str)
+        log_contents.append(log_content)
+
+    label_str = ""
+    if labels is not None:
+        label_str = json.dumps(labels)
+    return Log(contents=log_contents, labels=label_str)

huaweicloud_encryptionsdk/_lts_sdk/producer/model/result.py

@@ -0,0 +1,17 @@
+
+class Attempt(object):
+    def __init__(self, success, http_code, request_id, error_code, error_msg, time_stamp_ms, last_attempt_cost_ms):
+        self.success = success
+        self.http_code = http_code
+        self.request_id = request_id
+        self.error_code = error_code
+        self.error_msg = error_msg
+        self.time_stamp_ms = time_stamp_ms
+        self.last_attempt_cost_ms = last_attempt_cost_ms
+
+
+class Result(object):
+    def __init__(self):
+        self.attempt_list = []
+        self.success = False
+

huaweicloud_encryptionsdk/_lts_sdk/producer/model/retry_queue.py

@@ -0,0 +1,39 @@
+import queue
+import threading
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import log_batch
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class RetryQueue(object):
+
+    def __init__(self):
+        self._batch_queue = queue.Queue()
+        self._lock = threading.Lock()
+
+    def send_to_retry_queue(self, batch: log_batch.LogBatch):
+        self._lock.acquire()
+        try:
+            if batch is not None:
+                self._batch_queue.put_nowait(batch)
+        finally:
+            self._lock.release()
+
+    def get_retry_batch(self):
+        retry_batch_list = []
+        self._lock.acquire()
+        try:
+            while not self._batch_queue.empty():
+                retry_batch = self._batch_queue.get_nowait()
+                if retry_batch.next_retry_ms() < common.current_time_ms():
+                    retry_batch_list.append(retry_batch)
+                else:
+                    self._batch_queue.put_nowait(retry_batch)
+            return retry_batch_list
+        finally:
+            self._lock.release()
+
+    def length(self):
+        if self._batch_queue is None:
+            return 0
+        return self._batch_queue.qsize()