huaweicloud-encryption-sdk-python 1.2.1__py3-none-any.whl

huaweicloud_encryptionsdk/_lts_sdk/consumer/core/client_consumer_worker.py

@@ -0,0 +1,106 @@
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from loguru import logger
+
+from consumer.client.consumer_client_adapter import LogConsumerClientAdapter
+from consumer.core.consumer_heartbeat import LogConsumerHeartBeat
+from consumer.core.log_shard_consumer import LogShardConsumer
+from consumer.interface.consumer_processor_factory import ILogConsumerProcessorFactory
+from consumer.model.consumer_config import ConsumerConfig
+
+
+class ClientConsumerWorker(threading.Thread):
+    def __init__(self, processor_factory: ILogConsumerProcessorFactory, config: ConsumerConfig):
+        super().__init__()
+        self.processor_factory = processor_factory
+        self.config = config
+        self.executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="huawei-log-pool-")
+        self.shard_consumer = {}
+        self.client_adapter = LogConsumerClientAdapter(config)
+        self.heart_beat = LogConsumerHeartBeat(self.client_adapter, config)
+        self.shut_down = False
+        self.main_loop_exit = False
+
+    def check_consumer_group_exist(self):
+        err, check_point = self.client_adapter.fetch_consumer_group("")
+        return err
+
+    def run(self):
+        self.heart_beat.start()
+        while True:
+            if not self.shut_down:
+                self.heart_beat.lock.acquire_write()
+                for shard in self.heart_beat.current_held_shards:
+                    consumer = self.consumer_for_shard(shard)
+                    consumer.consume()
+
+                self.clean_consumer(self.heart_beat.current_held_shards)
+                self.heart_beat.lock.release_write()
+                time.sleep(self.config.fetch_interval_mill / 1000)
+            else:
+                break
+        self.main_loop_exit = True
+
+    def consumer_for_shard(self, shard_id):
+        if shard_id in self.shard_consumer:
+            return self.shard_consumer[shard_id]
+        return self.get_log_shard_consumer(shard_id)
+
+    def get_log_shard_consumer(self, shard_id):
+        consumer = LogShardConsumer(self.client_adapter, shard_id, self.processor_factory.generate_processor(),
+                                    self.config, self.heart_beat)
+        self.shard_consumer[shard_id] = consumer
+        logger.info("create a consumer for shard: {}", shard_id)
+        return consumer
+
+    def clean_consumer(self, owned_shard):
+        shard_to_unload = {}
+        for shard, consumer in self.shard_consumer.items():
+            if shard in owned_shard:
+                if consumer.is_shutdown():
+                    consumer = LogShardConsumer(self.client_adapter, shard,
+                                                self.processor_factory.generate_processor(),
+                                                self.config, self.heart_beat)
+                    logger.info("create a consumer for shard: {}", shard)
+                    self.shard_consumer[shard] = consumer
+                continue
+            # logger.info("shutting down consumer of shard: {}", shard)
+            # consumer.shutdown()
+            if consumer.is_shutdown():
+                shard_to_unload[shard] = consumer
+                logger.info("shard shutdown done, removing from heartbeat list: {}", shard)
+
+        for shard, consumer in shard_to_unload.items():
+            self.shard_consumer.pop(shard)
+
+        for shard, consumer in owned_shard.items():
+            if shard not in self.shard_consumer:
+                shard_to_unload[shard] = consumer
+
+        if len(shard_to_unload) > 0:
+            self.heart_beat.unsubscribe(shard_to_unload)
+            logger.warning("cancel heart beating for {}", shard_to_unload)
+
+    def shutdown(self):
+        self.shut_down = True
+        times = 0
+        while not self.main_loop_exit and times < 20:
+            times += 1
+            time.sleep(1)
+
+        for key, consumer in self.shard_consumer:
+            consumer.shutdown()
+
+        self.heart_beat.stop()
+        self.client_adapter.shutdown()
+
+
+def get_client_consumer_worker(processor_factory: ILogConsumerProcessorFactory, config: ConsumerConfig):
+    worker = ClientConsumerWorker(processor_factory, config)
+    err = worker.check_consumer_group_exist()
+    if err is not None:
+        logger.error("get client consumer work error {}", err.error_msg)
+        return None
+    return worker

huaweicloud_encryptionsdk/_lts_sdk/consumer/core/consumer_heartbeat.py

@@ -0,0 +1,73 @@
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+from loguru import logger
+
+from consumer.client.consumer_client_adapter import LogConsumerClientAdapter
+from consumer.model.consumer_config import ConsumerConfig
+from huaweicloud_encryptionsdk._lts_sdk.utils import common, lts_threading
+
+
+class LogConsumerHeartBeat(threading.Thread):
+    def __init__(self, client_adapter: LogConsumerClientAdapter, config: ConsumerConfig):
+        super().__init__()
+        self._client_adapter = client_adapter
+        self._heartbeat_interval_mill = config.heartbeat_interval_mill
+        self._timeout_sec = config.timeout_sec
+        self._current_held_shards = {}
+        self._all_heart_shards = {}
+        self._last_success_time = 0
+        self._is_stop = False
+        self._lock = lts_threading.ReadWriteLock()
+        self._thread_pool = ThreadPoolExecutor(max_workers=10240, thread_name_prefix="io_thread_pool_sender")
+
+    @property
+    def lock(self):
+        return self._lock
+
+    @property
+    def current_held_shards(self):
+        return self._current_held_shards
+
+    @current_held_shards.setter
+    def current_held_shards(self, value: int):
+        self._current_held_shards = value
+
+    def run(self):
+        while True:
+            if self._is_stop:
+                return
+            err, shards = self._client_adapter.hear_beat(_convert_all_heart_shards(self._all_heart_shards))
+            if err is not None:
+                now = common.current_time_ms()
+                if now - self._last_success_time > self._timeout_sec * 1000 + self._heartbeat_interval_mill:
+                    self._current_held_shards = {}
+            else:
+                self._current_held_shards = {}
+                self._lock.acquire_write()
+                for shard in shards:
+                    self._current_held_shards[shard] = ""
+                    self._all_heart_shards[shard] = ""
+                self._lock.release_write()
+                self._last_success_time = common.current_time_ms()
+                logger.debug(
+                    "after heart beat, current_held_shards is {}, all_heart_shards is {}, last_success_time is {}",
+                    self._current_held_shards, self._all_heart_shards, self._last_success_time)
+                common.sleep_ms(self._heartbeat_interval_mill)
+
+    def unsubscribe(self, shards):
+        for shard, consume in shards:
+            try:
+                self._all_heart_shards.pop(shard)
+            except KeyError as e:
+                logger.error("shard {} is not in all heart shard", shard)
+
+    def stop(self):
+        self._is_stop = True
+
+
+def _convert_all_heart_shards(all_heart_shards):
+    shards = []
+    for key, value in all_heart_shards.items():
+        shards.append(key)
+    return shards

huaweicloud_encryptionsdk/_lts_sdk/consumer/core/log_shard_consumer.py

@@ -0,0 +1,179 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from loguru import logger
+
+from consumer.client.consumer_client_adapter import LogConsumerClientAdapter
+from consumer.core.consumer_heartbeat import LogConsumerHeartBeat
+from consumer.interface.consumer_check_point_tracker import DefaultLogConsumerCheckPointTracker
+from consumer.interface.consumer_processor import ILogConsumerProcessor
+from consumer.model.consumer_config import ConsumerConfig
+from consumer.model.log_data import FetchedLogData
+from consumer.model.task_result import TaskResult, ProcessTaskResult
+from consumer.task.consumer_fetch_task import LogConsumerFetchTask, consumer_fetch_log
+from consumer.task.initial_task import InitializeTask, initialize_task
+from consumer.task.process_task import ProcessTask, process_task
+from consumer.task.shutdown_task import ShutDownTask, shutdown_task
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+INITIALIZING = "INITIALIZING"
+PROCESSING = "PROCESSING"
+SHUTTING_DOWN = "SHUTTING_DOWN"
+SHUTDOWN_COMPLETE = "SHUTDOWN_COMPLETE"
+
+
+class LogShardConsumer(object):
+    def __init__(self, client_adapter: LogConsumerClientAdapter, shard_id, processor: ILogConsumerProcessor,
+                 config: ConsumerConfig, heart_beat: LogConsumerHeartBeat):
+        self.client_adapter = client_adapter
+        self.shard_id = shard_id
+        self.start_time = config.start_time_ns
+        self.end_time = config.end_time_ns
+        self.processor = processor
+        self.checkpoint_tracker = DefaultLogConsumerCheckPointTracker(client_adapter=client_adapter, config=config,
+                                                                      shard_id=shard_id, heart_beat=heart_beat)
+        self.config = config
+        self.executor = ThreadPoolExecutor(max_workers=100, thread_name_prefix="log_shard_consumer-")
+        self.next_fetch_cursor = ""
+        self.final_fetch_cursor = ""
+        self.fetch_data_future = None
+        self.task_future = None
+        self.last_fetched_data = None
+        self.last_log_error_time = 0
+        self.current_status = INITIALIZING
+        self.current_task = None
+        self._shutdown = False
+        self.fetch_data_task_exist = False
+
+    def consume(self):
+        self.check_and_generate_next_task()
+        if self.current_status == PROCESSING and self.last_fetched_data is None:
+            self.fetch_data()
+
+    def do_process_task_result(self, process_task_result):
+        checkpoint = process_task_result.rollback_checkpoint
+        if checkpoint is not None and len(checkpoint) > 0:
+            self.cancel_current_fetch()
+            self.next_fetch_cursor = checkpoint
+
+    def do_process_init_result(self, init_task_result):
+        self.next_fetch_cursor = init_task_result.start_time
+        self.final_fetch_cursor = init_task_result.end_time
+        self.checkpoint_tracker.initial_cursor = self.next_fetch_cursor
+        if init_task_result.cursor_persistent:
+            self.checkpoint_tracker.last_save_check_point = self.next_fetch_cursor
+
+    def fetch_data(self):
+        has_error = False
+        if self.fetch_data_future is not None:
+            if self.fetch_data_future.cancelled():
+                self.fetch_data_future = None
+                self.last_fetched_data = None
+                return
+            elif not self.fetch_data_future.done():
+                return
+            result = self.get_task_result(self.fetch_data_future)
+            if result is not None and result.error is None:
+                fetch = result.fetch_data
+                self.last_fetched_data = FetchedLogData(self.shard_id, fetch, result.next_cursor,
+                                                        result.cursor)
+                self.next_fetch_cursor = result.next_cursor
+                self.sample_log_error(result)
+                has_error = result.error is not None
+        if not has_error:
+            task = LogConsumerFetchTask(self.client_adapter, self.shard_id, self.next_fetch_cursor,
+                                        self.final_fetch_cursor,
+                                        self.config)
+            self.fetch_data_future = self.executor.submit(consumer_fetch_log, task)
+        else:
+            self.fetch_data_future = None
+
+    def cancel_current_fetch(self):
+        if self.fetch_data_future is not None:
+            self.fetch_data_future.cancel()
+            self.get_task_result(self.fetch_data_future)
+            self.fetch_data_future = None
+            logger.info("cancel a fetch task, shard id: {}", self.shard_id)
+        self.last_fetched_data = None
+
+    def sample_log_error(self, result: TaskResult):
+        if result is not None and result.error is not None:
+            current_time = common.current_time_ms()
+            if current_time - self.last_log_error_time > 5 * 1000:
+                logger.warning("error is {}", result.error)
+                self.last_log_error_time = current_time
+
+    def update_status(self, task_success):
+        if self.current_status == SHUTTING_DOWN:
+            if self.current_task is None or task_success:
+                self.current_status = SHUTDOWN_COMPLETE
+        elif self.is_shutdown():
+            self.current_status = SHUTTING_DOWN
+        elif task_success:
+            if self.current_status == INITIALIZING:
+                self.current_status = PROCESSING
+
+    def generate_next_task(self):
+        next_task = None
+        logger.debug("when generate next task, status is {}", self.current_status)
+        if self.current_status == INITIALIZING:
+            next_task = InitializeTask(self.processor, self.client_adapter, self.shard_id, self.start_time,
+                                       self.end_time)
+        elif self.current_status == PROCESSING:
+            if self.last_fetched_data is not None:
+                self.checkpoint_tracker.current_cursor = self.last_fetched_data.cursor
+                self.checkpoint_tracker.next_cursor = self.last_fetched_data.next_cursor
+                next_task = ProcessTask(self.processor, self.last_fetched_data.fetchedData, self.checkpoint_tracker,
+                                        self.final_fetch_cursor)
+                self.last_fetched_data = None
+            else:
+                logger.debug("last fetched data is none")
+        elif self.current_status == SHUTTING_DOWN:
+            next_task = ShutDownTask(self.processor, self.checkpoint_tracker)
+            self.cancel_current_fetch()
+        if next_task is not None:
+            self.current_task = next_task
+            if isinstance(self.current_task, InitializeTask):
+                logger.debug("current task is initialize task")
+                self.task_future = self.executor.submit(initialize_task, self.current_task)
+            elif isinstance(self.current_task, ProcessTask):
+                logger.debug("current task is process task")
+                self.task_future = self.executor.submit(process_task, self.current_task)
+            elif isinstance(self.current_task, ShutDownTask):
+                logger.debug("current task is shutdown task")
+                self.task_future = self.executor.submit(shutdown_task, self.current_task)
+        else:
+            logger.debug("next task is none")
+
+    def check_and_generate_next_task(self):
+        if self.task_future is None or self.task_future.cancelled() or self.task_future.done():
+            task_success = False
+            result = self.get_task_result(self.task_future)
+            logger.debug("after get task result, result is {}", result)
+            self.task_future = None
+            if result is not None and result.error is None:
+                task_success = True
+                if self.current_status == INITIALIZING:
+                    self.do_process_init_result(result)
+                elif isinstance(result, ProcessTaskResult):
+                    self.do_process_task_result(result)
+            self.sample_log_error(result)
+            self.update_status(task_success)
+            self.generate_next_task()
+
+    def shutdown(self):
+        self._shutdown = True
+        if not self.is_shutdown():
+            self.check_and_generate_next_task()
+
+    def is_shutdown(self):
+        return self.current_status == SHUTDOWN_COMPLETE
+
+    @staticmethod
+    def get_task_result(future):
+        if future is not None and (future.done() or future.cancelled()):
+            try:
+                logger.debug("result type is {}", future)
+                return future.result()
+            except Exception as e:
+                logger.error("error retrieving task result, error: {}", e)
+        return None

huaweicloud_encryptionsdk/_lts_sdk/consumer/interface/consumer_check_point_tracker.py

@@ -0,0 +1,120 @@
+from consumer.client.consumer_client_adapter import LogConsumerClientAdapter
+from consumer.model.consumer_config import ConsumerConfig
+
+
+class ILogConsumerCheckPointTracker(object):
+    def save_check_point(self, persistent: bool):
+        raise NotImplemented("please implement me")
+
+    def get_check_point(self) -> str:
+        raise NotImplemented("please implement me")
+
+    def get_current_cursor(self) -> str:
+        raise NotImplemented("please implement me")
+
+
+class DefaultLogConsumerCheckPointTracker(ILogConsumerCheckPointTracker):
+
+    def __init__(self, client_adapter, config: ConsumerConfig, shard_id, heart_beat):
+        self._client_adapter = client_adapter
+        self._consumer = config.consumer
+        self._shard_id = shard_id
+        self._heart_beat = heart_beat
+
+        self._pending_check_point = ""
+        self._last_save_check_point = ""
+        self._next_cursor = ""
+        self._current_cursor = ""
+        self._initial_cursor = ""
+
+    @property
+    def client_adapter(self) -> LogConsumerClientAdapter:
+        return self._client_adapter
+
+    @property
+    def consumer(self):
+        return self._consumer
+
+    @consumer.setter
+    def consumer(self, value: str):
+        self._consumer = value
+
+    @property
+    def shard_id(self):
+        return self._shard_id
+
+    @shard_id.setter
+    def shard_id(self, value: str):
+        self._shard_id = value
+
+    @property
+    def pending_check_point(self):
+        return self._pending_check_point
+
+    @pending_check_point.setter
+    def pending_check_point(self, value: str):
+        self._pending_check_point = value
+
+    @property
+    def last_save_check_point(self):
+        return self._last_save_check_point
+
+    @last_save_check_point.setter
+    def last_save_check_point(self, value: str):
+        self._last_save_check_point = value
+
+    @property
+    def next_cursor(self):
+        return self._next_cursor
+
+    @next_cursor.setter
+    def next_cursor(self, value: str):
+        self._next_cursor = value
+
+    @property
+    def initial_cursor(self):
+        return self._initial_cursor
+
+    @initial_cursor.setter
+    def initial_cursor(self, value: str):
+        self._initial_cursor = value
+
+    @property
+    def current_cursor(self):
+        return self._current_cursor
+
+    @current_cursor.setter
+    def current_cursor(self, value: str):
+        self._current_cursor = value
+
+    def save_check_point(self, persistent: bool):
+        self._pending_check_point = self._next_cursor
+        if persistent:
+            return self.flush_check_point()
+        return None
+
+    def flush_check_point(self):
+        self._pending_check_point = self._next_cursor
+        checkpoint = self._pending_check_point
+        if isinstance(checkpoint, str):
+            if len(checkpoint) == 0 or checkpoint == self._last_save_check_point:
+                return None
+        elif isinstance(checkpoint, int):
+            if int == 0 or checkpoint == self._last_save_check_point:
+                return None
+        err = self._client_adapter.update_check_point(self._shard_id, self._consumer, checkpoint)
+        if err is None:
+            self._last_save_check_point = checkpoint
+        return err
+
+    def get_check_point(self) -> str:
+        if len(self._pending_check_point) != 0:
+            return self._pending_check_point
+        else:
+            return self._initial_cursor
+
+    def get_current_cursor(self) -> str:
+        return self._current_cursor
+
+    def consume_data_end_flush_checkpoint(self):
+        return self.flush_check_point()

huaweicloud_encryptionsdk/_lts_sdk/consumer/interface/consumer_processor.py

@@ -0,0 +1,18 @@
+from typing import List
+
+from consumer.interface.consumer_check_point_tracker import ILogConsumerCheckPointTracker
+from consumer.model.log_data import LogData
+
+
+class ILogConsumerProcessor(object):
+    def initialize(self, shard_id):
+        raise NotImplemented("please implement me")
+
+    def process(self, log_group: List[LogData], tracker: ILogConsumerCheckPointTracker) -> str:
+        raise NotImplemented("please implement me")
+
+    def shutdown(self, tracker: ILogConsumerCheckPointTracker):
+        raise NotImplemented("please implement me")
+
+    def shard_consumer_thread_success(self, shard_id, is_success):
+        pass

huaweicloud_encryptionsdk/_lts_sdk/consumer/interface/consumer_processor_factory.py

@@ -0,0 +1,3 @@
+class ILogConsumerProcessorFactory(object):
+    def generate_processor(self):
+        raise NotImplemented("please implement me")

huaweicloud_encryptionsdk/_lts_sdk/consumer/model/check_point.py

@@ -0,0 +1,20 @@
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class ShardCheckPoint:
+    def __init__(self):
+        self.shard_id = ""
+        self.checkpoint = 0
+        self.update_time = 0
+        self.consumer_name = ""
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self)
+
+
+class Cursor(object):
+    def __init__(self):
+        self.cursor = ""
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self)

huaweicloud_encryptionsdk/_lts_sdk/consumer/model/consumer_config.py

@@ -0,0 +1,27 @@
+import uuid
+
+
+class ConsumerConfig(object):
+    def __init__(self, endpoint, region_name, project_id, log_group_id, log_stream_id, access_key, access_secret,
+                 consumer_group_name, start_time_ns, end_time_ns, batch_size, heartbeat_interval_mill=1000,
+                 fetch_interval_mill=200, auto_commit_enable=True, auto_commit_interval_ms=60 * 1000, timeout_sec=60,
+                 unload_after_commit_enable=False):
+        self.endpoint = endpoint
+        self.region_name = region_name
+        self.project_id = project_id
+        self.log_group_id = log_group_id
+        self.log_stream_id = log_stream_id
+        self.access_key = access_key
+        self.access_secret = access_secret
+        self.consumer_group_name = consumer_group_name
+        self.start_time_ns = start_time_ns
+        self.end_time_ns = end_time_ns
+        self.consumer = str(uuid.uuid4())
+        self.is_compress = True
+        self.heartbeat_interval_mill = heartbeat_interval_mill
+        self.fetch_interval_mill = fetch_interval_mill
+        self.auto_commit_enable = auto_commit_enable
+        self.auto_commit_interval_ms = auto_commit_interval_ms
+        self.timeout_sec = timeout_sec
+        self.unload_after_commit_enable = unload_after_commit_enable
+        self.batch_size = batch_size

huaweicloud_encryptionsdk/_lts_sdk/consumer/model/log_data.py

@@ -0,0 +1,18 @@
+class LogData(object):
+    def __init__(self):
+        self.labels = {}
+
+
+class BatchGetLog(object):
+    def __init__(self):
+        self.count = 0
+        self.next = ""
+        self.logs = LogData()
+
+
+class FetchedLogData(object):
+    def __init__(self, shard_id, fetched_data, next_cursor, cursor):
+        self.shard_id = shard_id
+        self.cursor = cursor
+        self.next_cursor = next_cursor
+        self.fetchedData = fetched_data

huaweicloud_encryptionsdk/_lts_sdk/consumer/model/task_result.py

@@ -0,0 +1,40 @@
+from huaweicloud_encryptionsdk._lts_sdk.utils import common
+
+
+class TaskResult(object):
+    def __init__(self, error):
+        self.error = error
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self.error)
+
+
+class FetchTaskResult(TaskResult):
+    def __init__(self, fetch_data, cursor, next_cursor):
+        super().__init__(None)
+        self.fetch_data = fetch_data
+        self.cursor = cursor
+        self.next_cursor = next_cursor
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self)
+
+
+class InitTaskResult(TaskResult):
+    def __init__(self, start_time, end_time, cursor_persistent):
+        super().__init__(None)
+        self.start_time = start_time
+        self.end_time = end_time
+        self.cursor_persistent = cursor_persistent
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self)
+
+
+class ProcessTaskResult(TaskResult):
+    def __init__(self, rollback_checkpoint):
+        super().__init__(None)
+        self.rollback_checkpoint = rollback_checkpoint
+
+    def __str__(self):
+        return common.convert_obj2_json_str(self)

huaweicloud_encryptionsdk/_lts_sdk/consumer/task/consumer_fetch_task.py

@@ -0,0 +1,53 @@
+import time
+
+from loguru import logger
+
+from consumer.client.consumer_client_adapter import LogConsumerClientAdapter
+from consumer.model.consumer_config import ConsumerConfig
+from consumer.model.task_result import TaskResult, FetchTaskResult
+
+
+class LogConsumerFetchTask(object):
+    def __init__(self, client_adapter: LogConsumerClientAdapter, shard_id, cursor, final_fetch_cursor,
+                 config: ConsumerConfig):
+        self.client_adapter = client_adapter
+        self.shard_id = shard_id
+        self.cursor = cursor
+        self.final_fetch_cursor = final_fetch_cursor
+        self.config = config
+
+
+def consumer_fetch_log(task: LogConsumerFetchTask) -> TaskResult:
+    err = None
+    for i in range(2):
+        if task.final_fetch_cursor is not None and len(task.final_fetch_cursor) > 0:
+            err, batch_log = task.client_adapter.batch_get_logs(task.shard_id, task.config.batch_size, task.cursor,
+                                                                task.final_fetch_cursor)
+        else:
+            err, batch_log = task.client_adapter.batch_get_logs(task.shard_id, task.config.batch_size, task.cursor,
+                                                                "")
+        if err is not None:
+            logger.error("failed pull data from shard {}, cursor {}, error {}", task.shard_id, task.cursor, err)
+            time.sleep(0.2)
+            continue
+        else:
+            fetched_data = batch_log.logs
+            logger.debug("shard {}, cursor {}, next cursor {}, response size: {}", task.shard_id, task.cursor,
+                         batch_log.next, batch_log.count)
+            next_cursor = batch_log.next
+            if isinstance(next_cursor, int):
+                logger.debug("int cursor is {}", next_cursor)
+                if next_cursor == 0:
+                    logger.info("shard {} next cursor is empty, set to current int cursor {}", task.shard_id,
+                                task.cursor)
+                    next_cursor = task.cursor
+            if isinstance(next_cursor, str):
+                logger.debug("str cursor is {}", next_cursor)
+                if len(next_cursor) == 0:
+                    logger.info("shard {} next cursor is empty, set to current str cursor {}", task.shard_id,
+                                task.cursor)
+                    next_cursor = task.cursor
+            return FetchTaskResult(fetched_data, task.cursor, next_cursor)
+
+    if err is not None:
+        return TaskResult(err)