huaweicloud-encryption-sdk-python 1.2.1__py3-none-any.whl

huaweicloud_encryptionsdk/_lts_sdk/consumer/task/initial_task.py

@@ -0,0 +1,55 @@
+from loguru import logger
+
+from consumer.client.consumer_client_adapter import LogConsumerClientAdapter
+from consumer.interface.consumer_processor import ILogConsumerProcessor
+from consumer.model.task_result import TaskResult, InitTaskResult
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class InitializeTask(object):
+    def __init__(self, processor: ILogConsumerProcessor, client_adapter: LogConsumerClientAdapter, shard_id, start_time,
+                 end_time):
+        self.processor = processor
+        self.client_adapter = client_adapter
+        self.shard_id = shard_id
+        self.start_time = start_time
+        self.end_time = end_time
+
+
+def initialize_task(task: InitializeTask) -> TaskResult:
+    task.processor.initialize(task.shard_id)
+    err, shard_checkpoint_list = task.client_adapter.fetch_consumer_group(task.shard_id)
+    if err is not None:
+        logger.error("error fetching initial position: {}", err)
+        return TaskResult(err)
+    is_cursor_persistent = False
+    start_time_by_cursor = ""
+    end_time_by_cursor = ""
+    if task.end_time is not None and task.end_time != 0:
+        err, end_time_2_cursor = task.client_adapter.get_cursor_by_time(task.shard_id, str(task.end_time))
+        if err is not None:
+            logger.error("error fetching initial position, get cursor by time error: {}", err)
+            return TaskResult(err)
+        end_time_by_cursor = end_time_2_cursor.cursor
+        logger.debug("initialize task shardId {}, endTime {}, endTimeByCursor {}", task.shard_id, task.end_time,
+                     end_time_by_cursor)
+
+    logger.debug("shard checkpoint list is {}", shard_checkpoint_list)
+    if shard_checkpoint_list is not None and len(shard_checkpoint_list) > 0:
+        shard_checkpoint = shard_checkpoint_list[0]
+        logger.debug("shard checkpoint is {}", shard_checkpoint)
+        is_cursor_persistent = True
+        start_time_by_cursor = shard_checkpoint.checkpoint
+    else:
+        if task.start_time is not None and task.start_time != 0:
+            err, start_time_2_cursor = task.client_adapter.get_cursor_by_time(task.shard_id, str(task.start_time))
+            if err is not None:
+                logger.error("error fetching initial position, get cursor by time error: {}", err)
+                return TaskResult(err)
+            start_time_by_cursor = start_time_2_cursor.cursor
+            logger.debug("initialize task shardId {}, endTime {}, endTimeByCursor {}", task.shard_id, task.start_time,
+                         start_time_by_cursor)
+
+    logger.debug("initialize result, shardId {}, startTimeByCursor {}, endTimeByCursor {}, isCursorPersistent {}",
+                 task.shard_id, start_time_by_cursor, end_time_by_cursor, is_cursor_persistent)
+    return InitTaskResult(start_time_by_cursor, end_time_by_cursor, is_cursor_persistent)

huaweicloud_encryptionsdk/_lts_sdk/consumer/task/process_task.py

@@ -0,0 +1,25 @@
+from consumer.interface.consumer_check_point_tracker import DefaultLogConsumerCheckPointTracker
+from consumer.interface.consumer_processor import ILogConsumerProcessor
+from consumer.model.task_result import TaskResult, ProcessTaskResult
+
+
+class ProcessTask(object):
+    def __init__(self, processor: ILogConsumerProcessor, log_groups,
+                 checkpoint_tracker: DefaultLogConsumerCheckPointTracker,
+                 final_fetch_cursor):
+        self.processor = processor
+        self.chunk = log_groups
+        self.checkpoint_tracker = checkpoint_tracker
+        self.final_fetch_cursor = final_fetch_cursor
+
+
+def process_task(task: ProcessTask) -> TaskResult:
+    task.checkpoint_tracker.pending_check_point = task.checkpoint_tracker.next_cursor
+    checkpoint = task.processor.process(task.chunk, task.checkpoint_tracker)
+    task.processor.shard_consumer_thread_success(task.checkpoint_tracker.shard_id,
+                                                 task.final_fetch_cursor == task.checkpoint_tracker.next_cursor)
+    err = task.checkpoint_tracker.consume_data_end_flush_checkpoint()
+    if err is None:
+        return ProcessTaskResult(checkpoint)
+    else:
+        return TaskResult(err)

huaweicloud_encryptionsdk/_lts_sdk/consumer/task/shutdown_task.py

@@ -0,0 +1,24 @@
+from loguru import logger
+
+from consumer.interface.consumer_check_point_tracker import DefaultLogConsumerCheckPointTracker
+from consumer.interface.consumer_processor import ILogConsumerProcessor
+from consumer.model.task_result import TaskResult
+
+
+class ShutDownTask(object):
+    def __init__(self, processor: ILogConsumerProcessor, checkpoint_tracker: DefaultLogConsumerCheckPointTracker):
+        self.processor = processor
+        self.checkpoint_tracker = checkpoint_tracker
+
+
+def shutdown_task(task: ShutDownTask):
+    try:
+        task.processor.shutdown(task.checkpoint_tracker)
+    except Exception as e:
+        logger.error("could not shut down processor {}", e)
+
+    err = task.checkpoint_tracker.flush_check_point()
+    if err is not None:
+        return TaskResult(None)
+    else:
+        return TaskResult(err)

huaweicloud_encryptionsdk/_lts_sdk/consumer_main.py

@@ -0,0 +1,88 @@
+import argparse
+import sys
+import time
+from typing import List
+
+from loguru import logger
+
+from consumer.core.client_consumer_worker import get_client_consumer_worker
+from consumer.interface.consumer_check_point_tracker import ILogConsumerCheckPointTracker
+from consumer.interface.consumer_processor import ILogConsumerProcessor
+from consumer.interface.consumer_processor_factory import ILogConsumerProcessorFactory
+from consumer.model.consumer_config import ConsumerConfig
+from consumer.model.log_data import LogData
+
+
+class DemoLogConsumerProcessor(ILogConsumerProcessor):
+
+    def __init__(self):
+        self.log_count = 0
+
+    def initialize(self, shard_id):
+        pass
+
+    def process(self, log_group: List[LogData], tracker: ILogConsumerCheckPointTracker) -> str:
+        self.log_count += len(log_group)
+        logger.info("this time consume log {} total log num {}", len(log_group), self.log_count)
+
+    def shutdown(self, tracker: ILogConsumerCheckPointTracker):
+        return tracker.save_check_point(True)
+
+
+class DemoLogConsumerProcessorFactory(ILogConsumerProcessorFactory):
+    def generate_processor(self):
+        return DemoLogConsumerProcessor()
+
+
+def start_consume(endpoint, region_name, project_id, log_group_id, log_stream_id, access_key, access_secret,
+                  consumer_group_name, start_time, end_time, log_level, is_test, consumer_count, consume_batch_size):
+    logger.remove()
+    logger.add(sys.stderr, level=log_level)
+    if is_test:
+        logger.add("/opt/cloud/log.log", level="DEBUG")
+    logger.info("start time {}, end time {}", start_time, end_time)
+    workers = []
+
+    for i in range(consumer_count):
+        config = ConsumerConfig(endpoint, region_name, project_id, log_group_id, log_stream_id, access_key,
+                                access_secret, consumer_group_name, start_time, end_time, consume_batch_size)
+        work = get_client_consumer_worker(DemoLogConsumerProcessorFactory(), config)
+        if work is None:
+            logger.error("get work error, work is none")
+        else:
+            workers.append(work)
+
+    for work in workers:
+        work.start()
+
+    time.sleep(30000 * 60)
+    for work in workers:
+        work.shutdown()
+
+    time.sleep(60)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='lts python sdk consume log help tester information')
+    parser.add_argument('--endpoint', dest='endpoint', type=str, help='lts endpoint', default="https://endpoint")
+    parser.add_argument('--project_id', dest='project_id', type=str, help='huaweicloud project_id', default="projectId")
+    parser.add_argument('--region', dest='region', type=str, help='huaweicloud region', default="region")
+    parser.add_argument('--group_id', dest='group_id', type=str, help='send log group_id', default="group_id")
+    parser.add_argument('--stream_id', dest='stream_id', type=str, help='send log stream_id', default="stream_id")
+    parser.add_argument('--ak', dest='ak', type=str, help='huaweicloud ak', default="ak")
+    parser.add_argument('--sk', dest='sk', type=str, help='huaweicloud sk', default="sk")
+    parser.add_argument('--consumer_count', dest='consumer_count', type=int, help='consumer count', default=1)
+    parser.add_argument('--consume_batch_size', dest='consume_batch_size', type=int, help='each fetch max log nums',
+                        default=1000)
+    parser.add_argument('--consumer_group_name', dest='consumer_group_name', type=str, help='consumer group name',
+                        default="consumer_group_name")
+    parser.add_argument('--start_time', dest='start_time', type=int, help='consume log start time ms',
+                        default=213123123123)
+    parser.add_argument('--end_time', dest='end_time', type=int, help='consume log end time ms', default=0)
+    parser.add_argument('--log_level', dest='log_level', type=str, help='consume log end time ms', default="INFO")
+    parser.add_argument('--is_test', dest='is_test', type=bool, help='test consumer will write log to node',
+                        default=True)
+    args = parser.parse_args()
+    start_consume(args.endpoint, args.region, args.project_id, args.group_id, args.stream_id, args.ak,
+                  args.sk, args.consumer_group_name, args.start_time, args.end_time, args.log_level, args.is_test,
+                  args.consumer_count, args.consume_batch_size)

huaweicloud_encryptionsdk/_lts_sdk/parallel_main.py

@@ -0,0 +1,64 @@
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from loguru import logger
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.core.producer import Producer
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import lts_store
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.config import Config
+from huaweicloud_encryptionsdk._lts_sdk.producer.utils import common
+
+
+def generate_random_str(str_len=16):
+    """
+    生成一个指定长度的随机字符串
+    """
+    random_str = ''
+    base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789'
+    length = len(base_str) - 1
+    for i in range(str_len):
+        random_str += base_str[random.randint(0, length)]
+    return random_str
+
+def send_log_single_producer():
+    thread_num = 100
+    threads = []
+    begin_ms = common.current_time_ms()
+    submit_send_log_thread_pool = ThreadPoolExecutor(max_workers=thread_num, thread_name_prefix="send_log_thread")
+
+    config = Config()
+    config.endpoint = ""
+    config.access_key = ""
+    config.access_secret = ""
+    config.region_id = ""
+    config.project_id = ""
+    producer = Producer.init_producer(config)
+    producer.start_producer()
+
+    group_id = ""
+    stream_id = ""
+
+    log_content = generate_random_str(1024)
+    log = [log_content]
+    labels = {"keyA": "valueA"}
+    log_p = lts_store.generate_log(log, labels)
+
+    for i in range(thread_num):
+        new_thread = submit_send_log_thread_pool.submit(send_log_function, producer, group_id, stream_id, log_p)
+        threads.append(new_thread)
+
+    for t in threads:
+        t.result()
+    end_ms = common.current_time_ms()
+    logger.info("end send log, cost [{}]ms", end_ms - begin_ms)
+
+
+def send_log_function(one_producer, group_id, stream_id, log_p):
+    for i in range(2000):
+        one_producer.send_log(group_id, stream_id, log_p)
+
+
+if __name__ == '__main__':
+    send_log_single_producer()
+    time.sleep(1000)

huaweicloud_encryptionsdk/_lts_sdk/producer/callback/call_back.py

@@ -0,0 +1,7 @@
+class CallBack(object):
+
+    def invoke_on_success(self, result):
+        pass
+
+    def invoke_on_failed(self, result):
+        pass

huaweicloud_encryptionsdk/_lts_sdk/producer/core/accumulator.py

@@ -0,0 +1,135 @@
+from loguru import logger
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import log_batch
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.log import Log
+from huaweicloud_encryptionsdk._lts_sdk.utils import lts_threading
+
+DELIMITER = "|"
+
+log_num = 0
+
+
+class Accumulator(object):
+
+    def __init__(self):
+        self._log_batch_map = {}
+        self._accumulator_lock = lts_threading.ReadWriteLock()
+        self._config = None
+        self._io_worker = None
+        self._producer = None
+        self._io_thread_pool = None
+        self._already_send_num = 0
+
+    @staticmethod
+    def init_accumulator(config, io_worker, producer, thread_pool):
+        accumulator = Accumulator()
+        accumulator.config = config
+        accumulator.io_thread_pool = thread_pool
+        accumulator.io_worker = io_worker
+        accumulator.producer = producer
+        return accumulator
+
+    @property
+    def log_batch_map(self):
+        return self._log_batch_map
+
+    @property
+    def accumulator_lock(self):
+        return self._accumulator_lock
+
+    @log_batch_map.setter
+    def log_batch_map(self, value):
+        self._log_batch_map = value
+
+    @property
+    def config(self):
+        return self._config
+
+    @config.setter
+    def config(self, value):
+        self._config = value
+
+    @property
+    def io_worker(self):
+        return self._io_worker
+
+    @io_worker.setter
+    def io_worker(self, value):
+        self._io_worker = value
+
+    @property
+    def io_thread_pool(self):
+        return self._io_thread_pool
+
+    @io_thread_pool.setter
+    def io_thread_pool(self, value):
+        self._io_thread_pool = value
+
+    @property
+    def producer(self):
+        return self._producer
+
+    @producer.setter
+    def producer(self, value):
+        self._producer = value
+
+    def add_log_to_log_batch_map(self, log_data: Log, group_id: str, stream_id: str, call_back):
+        global log_num
+        try:
+            key = self.get_log_batch_key(group_id, stream_id)
+            self._accumulator_lock.acquire_write()
+            try:
+                if key in self.log_batch_map:
+                    batch_value = self.log_batch_map.get(key)
+                    # TODO 需要计算producer的总大小
+                    log_size = log_data.getSize()
+                    batch_value.add_total_data_size(log_size)
+                    self._add_or_send_producer_batch(key, group_id, stream_id, batch_value, log_data, call_back)
+
+                else:
+                    self._create_new_producer_log_batch(log_data, key, group_id, stream_id, call_back)
+
+            finally:
+                self._accumulator_lock.release_write()
+
+            return None
+        finally:
+            log_num += len(log_data.contents)
+            if log_num % 10000 == 0:
+                logger.warning("add log num {}", log_num)
+
+    def _create_new_producer_log_batch(self, log_data: Log, key: str, group_id: str, stream_id: str, call_back):
+        _new_batch = log_batch.init_product_batch_var_single_log(log_data, call_back, group_id, stream_id, self.config)
+        self.log_batch_map[key] = _new_batch
+
+    def _add_or_send_producer_batch(self, key: str, group_id: str, stream_id: str, batch_value: log_batch.LogBatch,
+                                    log: Log,
+                                    call_back):
+        total_data_count = batch_value.log_group_count + 1
+        # batch中日志大小超过MaxBatchSize && batch中日志大小< 5242880 && batch中日志条数 <= MaxBatchCount
+        # 直接发送
+        if (self.config.batch_size_threshold_in_bytes < batch_value.total_data_size < 5242880
+                and total_data_count <= self.config.batch_count_threshold):
+            batch_value.add_log_to_log_group(log)
+            if call_back is not None:
+                batch_value.add_call_back(call_back)
+            self._inner_send_to_server(key, batch_value)
+        # batch中日志大小和条数都没有满足 配置，则log加入batch中先缓存
+        elif (batch_value.total_data_size <= self.config.batch_size_threshold_in_bytes
+              and total_data_count <= self.config.batch_count_threshold):
+
+            batch_value.add_log_to_log_group(log)
+            if call_back is not None:
+                batch_value.add_call_back(call_back)
+
+        else:
+            self._inner_send_to_server(key, batch_value)
+            self._create_new_producer_log_batch(log, key, group_id, stream_id, call_back)
+
+    def _inner_send_to_server(self, key: str, batch: log_batch.LogBatch):
+        self.io_thread_pool.add_task(batch)
+        self.log_batch_map.pop(key)
+
+    @staticmethod
+    def get_log_batch_key(group_id: str, stream_id: str) -> str:
+        return group_id + DELIMITER + stream_id + DELIMITER

huaweicloud_encryptionsdk/_lts_sdk/producer/core/io_thread_pool.py

@@ -0,0 +1,107 @@
+import queue
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from loguru import logger
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.log_batch import LogBatch
+from huaweicloud_encryptionsdk._lts_sdk.utils import lts_threading
+
+
+class IOThreadPool(threading.Thread):
+    def __init__(self, io_worker):
+        super().__init__()
+        self._thread_pool_lock = lts_threading.ReadWriteLock()
+        self._deal_batch_queue = queue.Queue()
+        self._submit_thread_pool = ThreadPoolExecutor(max_workers=10240, thread_name_prefix="io_thread_pool_sender")
+        self._io_worker = io_worker
+
+    @property
+    def io_worker(self):
+        return self._io_worker
+
+    @io_worker.setter
+    def io_worker(self, value: int):
+        self._io_worker = value
+
+    @property
+    def thread_pool_lock(self):
+        return self._thread_pool_lock
+
+    @thread_pool_lock.setter
+    def thread_pool_lock(self, value: int):
+        self._thread_pool_lock = value
+
+    @property
+    def deal_batch_queue(self):
+        self._thread_pool_lock.acquire_read()
+        try:
+            return self._deal_batch_queue
+        finally:
+            self._thread_pool_lock.release_read()
+
+    @deal_batch_queue.setter
+    def deal_batch_queue(self, value: int):
+        self._thread_pool_lock.acquire_write()
+        try:
+            self._deal_batch_queue = value
+        finally:
+            self._thread_pool_lock.release_write()
+
+    def add_task(self, batch: LogBatch):
+
+        self._thread_pool_lock.acquire_write()
+        try:
+            self._deal_batch_queue.put_nowait(batch)
+        except queue.Full as e:
+            logger.error("log batch queue is full, log generate is too fast")
+        finally:
+            self._thread_pool_lock.release_write()
+
+    def take_task(self):
+        self._thread_pool_lock.acquire_read()
+        try:
+            if self._deal_batch_queue is None or self._deal_batch_queue.empty():
+                return None
+            batch = self._deal_batch_queue.get_nowait()
+            return batch
+        finally:
+            self._thread_pool_lock.release_read()
+
+    def has_task(self):
+        self._thread_pool_lock.acquire_read()
+        try:
+            if self._deal_batch_queue is None:
+                return False
+            return not self._deal_batch_queue.empty()
+        finally:
+            self._thread_pool_lock.release_read()
+
+    def task_queue_size(self):
+        self._thread_pool_lock.acquire_read()
+        try:
+            if self._deal_batch_queue is None:
+                return 0
+            return self._deal_batch_queue.qsize()
+        finally:
+            self._thread_pool_lock.release_read()
+
+    def run(self):
+        try:
+            while True:
+                batch = self.take_task()
+                if batch is None:
+                    time.sleep(0.1)
+                else:
+                    self.io_worker.start_this_send_task()
+
+                    def send_to_server_handler(io_worker, log_batch):
+                        try:
+                            io_worker.io_worker_send_to_server(log_batch)
+                        finally:
+                            io_worker.end_this_send_task()
+
+                    self._submit_thread_pool.submit(send_to_server_handler, self.io_worker, batch)
+
+        finally:
+            self.io_worker.dec_task_count()

huaweicloud_encryptionsdk/_lts_sdk/producer/core/io_worker.py

@@ -0,0 +1,124 @@
+import math
+import queue
+import threading
+
+from huaweicloud_encryptionsdk._lts_sdk.producer.lts_http import http_client
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import log_batch
+from huaweicloud_encryptionsdk._lts_sdk.producer.model import result
+from huaweicloud_encryptionsdk._lts_sdk.producer.model.error import Error
+from huaweicloud_encryptionsdk._lts_sdk.utils import lts_threading, common
+
+
+class IOWorker(threading.Thread):
+    def __init__(self, max_io_worker_count, no_retry_code, producer):
+        super().__init__()
+        self._task_count_lock = lts_threading.ReadWriteLock()
+        self._task_count = 0
+        # 控制io_worker数量，超过限制时，后面的都进行等待，等待前面io_worker处理完
+        self._max_io_worker = queue.Queue(max_io_worker_count)
+        self._no_retry_status_code = no_retry_code
+        self._producer = producer
+        self._retry_queue_lock = lts_threading.ReadWriteLock()
+        self._retry_queue = queue.Queue()
+        self._lock = threading.Lock()
+        self._client = http_client.HttpClient()
+
+    @property
+    def producer(self):
+        return self._producer
+
+    @producer.setter
+    def producer(self, value):
+        self._producer = value
+
+    @property
+    def task_count(self):
+        self._task_count_lock.acquire_read()
+        try:
+            return self._task_count
+        finally:
+            self._task_count_lock.release_read()
+
+    @task_count.setter
+    def task_count(self, value):
+        self._task_count_lock.acquire_write()
+        try:
+            self._task_count = value
+        finally:
+            self._task_count_lock.release_write()
+
+    def io_worker_send_to_server(self, batch: log_batch.LogBatch):
+        begin_time = common.current_time_ms()
+        # TODO 此处进行日志发送
+        err = self._client.put_logs(batch.group_id, batch.stream_id, batch.log_group, self._producer.producer_config)
+        if err is None:
+            if batch.attempt_count < self._producer.producer_config.max_reserved_attempts:
+                now_time = common.current_time_ms()
+                attempt = result.Attempt(True, 200, "", "", "", now_time, now_time - begin_time)
+                batch.add_attempt(attempt)
+            batch.result.success = True
+            if len(batch.call_back_list) > 0:
+                for call_back in batch.call_back_list:
+                    call_back.invoke_on_success(batch.result)
+
+        else:
+            if err.http_code in self._no_retry_status_code:
+                self.add_error_msg_to_batch_attempt(batch, err, False, begin_time)
+                self.exec_failed_call_back(batch)
+                return
+
+            if batch.attempt_count < self._producer.producer_config.retries:
+                self.add_error_msg_to_batch_attempt(batch, err, True, begin_time)
+                retry_wait_time = self._producer.producer_config.base_retry_backoff_ms * math.pow(2,
+                                                                                                  batch.attempt_count - 1)
+                if retry_wait_time < self.producer.producer_config.max_retry_backoff_ms:
+                    batch.next_retry_ms = common.current_time_ms() + retry_wait_time
+                else:
+                    batch.next_retry_ms = common.current_time_ms() + self.producer.producer_config.max_retry_backoff_ms
+                self.send_to_retry_queue(batch)
+            else:
+                self.exec_failed_call_back(batch)
+
+    def end_this_send_task(self):
+        self._max_io_worker.get()
+        self.dec_task_count()
+
+    def start_this_send_task(self):
+        self.inc_task_count()
+        self._max_io_worker.put(1)
+
+    def inc_task_count(self):
+        self._task_count_lock.acquire_write()
+        try:
+            self._task_count += 1
+        finally:
+            self._task_count_lock.release_write()
+
+    def dec_task_count(self):
+        self._task_count_lock.acquire_write()
+        try:
+            self._task_count -= 1
+        finally:
+            self._task_count_lock.release_write()
+
+    def send_to_retry_queue(self, batch: log_batch.LogBatch):
+        self._retry_queue_lock.acquire_write()
+        try:
+            self._retry_queue.put_nowait(batch)
+        finally:
+            self._retry_queue_lock.release_write()
+
+    def add_error_msg_to_batch_attempt(self, batch: log_batch.LogBatch, err: Error, retry: bool, begin_ms: int):
+        if batch.attempt_count < self.producer.producer_config.max_reserved_attempts:
+            now_ms = common.current_time_ms()
+            attempt = result.Attempt(False, err.http_code, err.request_id, err.error_code, err.error_msg, now_ms,
+                                     now_ms - begin_ms)
+            batch.add_attempt(attempt)
+        batch.result.success = False
+        batch.attempt_count += 1
+
+    @classmethod
+    def exec_failed_call_back(cls, batch: log_batch.LogBatch):
+        if len(batch.call_back_list) > 0:
+            for call_back in batch.call_back_list:
+                call_back.invoke_on_failed(batch.result)