cobweb-launcher 3.1.8__py3-none-any.whl → 3.1.9__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -128,7 +128,7 @@ class Launcher:
                     self.__WORKER_THREAD__[name] = worker_thread
                     worker_thread.start()
                 time.sleep(3)
-        logger.info("monitor thread close ...")
+        logger.info("main thread close!")
 
     def start(self):
         self._pause.is_set()

cobweb/schedulers/scheduler_with_redis.py

@@ -147,7 +147,6 @@ class RedisScheduler(Scheduler):
                     elif todo_count:
                         logger.info(f"Recovery {self.task} task run！todo seeds count: {todo_count}, queue length: {all_count}")
                         self.pause.clear()
-                        # self.execute()
                     else:
                         logger.info("pause! waiting for resume...")
                 else:
@@ -173,5 +172,5 @@ class RedisScheduler(Scheduler):
 
             time.sleep(30)
 
-        logger.info("Done! Ready to close thread...")
+        logger.info("Scheduler Done!")
 

cobweb/utils/__init__.py

@@ -1,4 +1,4 @@
-from .oss import OssUtil
+# from .oss import OssUtil
 from .tools import *
 from .bloom import BloomFilter
 from .dotting import LoghubDot

cobweb/utils/oss.py

@@ -1,113 +1,113 @@
-
-from cobweb import setting
-from requests import Response
-from oss2 import Auth, Bucket, models, PartIterator
-from cobweb.exceptions import oss_db_exception
-from cobweb.utils.decorators import decorator_oss_db
-
-
-class OssUtil:
-
-    def __init__(
-        self,
-        bucket=None,
-        endpoint=None,
-        access_key=None,
-        secret_key=None,
-        chunk_size=None,
-        min_upload_size=None,
-        **kwargs
-    ):
-        self.bucket = bucket or setting.OSS_BUCKET
-        self.endpoint = endpoint or setting.OSS_ENDPOINT
-        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
-        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
-
-        self.failed_count = 0
-        self._kw = kwargs
-
-        self._auth = Auth(
-            access_key_id=access_key or setting.OSS_ACCESS_KEY,
-            access_key_secret=secret_key or setting.OSS_SECRET_KEY
-        )
-        self._client = Bucket(
-            auth=self._auth,
-            endpoint=self.endpoint,
-            bucket_name=self.bucket,
-            **self._kw
-        )
-
-    def failed(self):
-        self.failed_count += 1
-        if self.failed_count >= 5:
-            self._client = Bucket(
-                auth=self._auth,
-                endpoint=self.endpoint,
-                bucket_name=self.bucket,
-                **self._kw
-            )
-
-    def exists(self, key: str) -> bool:
-        try:
-            result = self._client.object_exists(key)
-            self.failed_count = 0
-            return result
-        except Exception as e:
-            self.failed()
-            raise e
-
-    def head(self, key: str) -> models.HeadObjectResult:
-        return self._client.head_object(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
-    def init_part(self, key) -> models.InitMultipartUploadResult:
-        """初始化分片上传"""
-        return self._client.init_multipart_upload(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
-    def put(self, key, data) -> models.PutObjectResult:
-        """文件上传"""
-        return self._client.put_object(key, data)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
-    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
-        """分片上传"""
-        return self._client.upload_part(key, upload_id, position, data)
-
-    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
-        """获取分片列表"""
-        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
-    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
-        """合并分片"""
-        headers = None if parts else {"x-oss-complete-all": "yes"}
-        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
-    def append(self, key, position, data) -> models.AppendObjectResult:
-        """追加上传"""
-        return self._client.append_object(key, position, data)
-
-    def iter_data(self, data, chunk_size=None):
-        chunk_size = chunk_size or self.chunk_size
-        if isinstance(data, Response):
-            for part_data in data.iter_content(chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), chunk_size):
-                yield data[i:i + chunk_size]
-
-    def assemble(self, ready_data, data, chunk_size=None):
-        upload_data = b""
-        ready_data = ready_data + data
-        chunk_size = chunk_size or self.chunk_size
-        if len(ready_data) >= chunk_size:
-            upload_data = ready_data[:chunk_size]
-            ready_data = ready_data[chunk_size:]
-        return ready_data, upload_data
-
-    def content_length(self, key: str) -> int:
-        head = self.head(key)
-        return head.content_length
-
+#
+# from cobweb import setting
+# from requests import Response
+# from oss2 import Auth, Bucket, models, PartIterator
+# from cobweb.exceptions import oss_db_exception
+# from cobweb.utils.decorators import decorator_oss_db
+#
+#
+# class OssUtil:
+#
+#     def __init__(
+#         self,
+#         bucket=None,
+#         endpoint=None,
+#         access_key=None,
+#         secret_key=None,
+#         chunk_size=None,
+#         min_upload_size=None,
+#         **kwargs
+#     ):
+#         self.bucket = bucket or setting.OSS_BUCKET
+#         self.endpoint = endpoint or setting.OSS_ENDPOINT
+#         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
+#         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+#
+#         self.failed_count = 0
+#         self._kw = kwargs
+#
+#         self._auth = Auth(
+#             access_key_id=access_key or setting.OSS_ACCESS_KEY,
+#             access_key_secret=secret_key or setting.OSS_SECRET_KEY
+#         )
+#         self._client = Bucket(
+#             auth=self._auth,
+#             endpoint=self.endpoint,
+#             bucket_name=self.bucket,
+#             **self._kw
+#         )
+#
+#     def failed(self):
+#         self.failed_count += 1
+#         if self.failed_count >= 5:
+#             self._client = Bucket(
+#                 auth=self._auth,
+#                 endpoint=self.endpoint,
+#                 bucket_name=self.bucket,
+#                 **self._kw
+#             )
+#
+#     def exists(self, key: str) -> bool:
+#         try:
+#             result = self._client.object_exists(key)
+#             self.failed_count = 0
+#             return result
+#         except Exception as e:
+#             self.failed()
+#             raise e
+#
+#     def head(self, key: str) -> models.HeadObjectResult:
+#         return self._client.head_object(key)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+#     def init_part(self, key) -> models.InitMultipartUploadResult:
+#         """初始化分片上传"""
+#         return self._client.init_multipart_upload(key)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+#     def put(self, key, data) -> models.PutObjectResult:
+#         """文件上传"""
+#         return self._client.put_object(key, data)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+#     def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
+#         """分片上传"""
+#         return self._client.upload_part(key, upload_id, position, data)
+#
+#     def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
+#         """获取分片列表"""
+#         return [part_info for part_info in PartIterator(self._client, key, upload_id)]
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+#     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
+#         """合并分片"""
+#         headers = None if parts else {"x-oss-complete-all": "yes"}
+#         return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+#     def append(self, key, position, data) -> models.AppendObjectResult:
+#         """追加上传"""
+#         return self._client.append_object(key, position, data)
+#
+#     def iter_data(self, data, chunk_size=None):
+#         chunk_size = chunk_size or self.chunk_size
+#         if isinstance(data, Response):
+#             for part_data in data.iter_content(chunk_size):
+#                 yield part_data
+#         if isinstance(data, bytes):
+#             for i in range(0, len(data), chunk_size):
+#                 yield data[i:i + chunk_size]
+#
+#     def assemble(self, ready_data, data, chunk_size=None):
+#         upload_data = b""
+#         ready_data = ready_data + data
+#         chunk_size = chunk_size or self.chunk_size
+#         if len(ready_data) >= chunk_size:
+#             upload_data = ready_data[:chunk_size]
+#             ready_data = ready_data[chunk_size:]
+#         return ready_data, upload_data
+#
+#     def content_length(self, key: str) -> int:
+#         head = self.head(key)
+#         return head.content_length
+#

{cobweb_launcher-3.1.8.dist-info → cobweb_launcher-3.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.8
+Version: 3.1.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.8.dist-info → cobweb_launcher-3.1.9.dist-info}/RECORD

@@ -17,25 +17,23 @@ cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
 cobweb/launchers/distributor.py,sha256=yZJ7d8EXZGukRr8SzcxXZu1z37VxKqBWE1apNW9T9uE,6429
-cobweb/launchers/launcher.py,sha256=zP0l-5QXxoZauh2bnIuHUJ6L5QI0bbgXvhEN49Ff43g,5350
+cobweb/launchers/launcher.py,sha256=HJJgVIh2vAfG2wnU_yHEN_v1tT_qpz2F4JmkcBoZvCE,5344
 cobweb/launchers/uploader.py,sha256=ePQsGxoFw3Le4DgkoDGDqxrv8lGaYKCnGq2nuaWba9o,1771
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=qwoOYMhlAB-MnEmMNpNeauTHoRTOr2wyBDYS4MF6B1c,261
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
-cobweb/schedulers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
-cobweb/schedulers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
 cobweb/schedulers/scheduler.py,sha256=mN9XvaOCzNnBWQfzslTIM860ZGq2gyLtxpbVUd0Slqs,2240
-cobweb/schedulers/scheduler_with_redis.py,sha256=Qq8iMxbDWJpRml76NfcZyyB8q7tM2Et0YrOcF7nwcP0,6399
-cobweb/utils/__init__.py,sha256=8Bu5iZrIOUMS4jv4hi0inRPtscf6MK0ZFa7gQ7ZFoqw,145
+cobweb/schedulers/scheduler_with_redis.py,sha256=EUsewtCDxgtgNgv2ogQus56GNv9IujkNLcpki-LPQzs,6343
+cobweb/utils/__init__.py,sha256=yqIU08RcBRa2JZDr1PsrnXl473Ni5JqhagWIQvyGsc8,147
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
 cobweb/utils/dotting.py,sha256=x34WryNKu_o54EzVwAZByagjMTXF0ZP0VYKdjfYEdSM,1833
-cobweb/utils/oss.py,sha256=ktfwMGnq5FMzOkUUS3nNXI7pTdPNinudH3YRJ0nMhoU,3985
+cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-3.1.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.8.dist-info/METADATA,sha256=dHvMv0I_533y0KCp5RSVT87mqbznPgWakYtN3OeN3KQ,6509
-cobweb_launcher-3.1.8.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.8.dist-info/RECORD,,
+cobweb_launcher-3.1.9.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.9.dist-info/METADATA,sha256=eCmqW-nWkaL_ez6C7aTCw7u-gcTsYd3Y6b3IK6I0akY,6509
+cobweb_launcher-3.1.9.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.9.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.9.dist-info/RECORD,,

cobweb/schedulers/launcher_air.py

@@ -1,93 +0,0 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        new_seeds = {}
-        del_seeds = set()
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed_tuple:
-                break
-            seed, new_seed = seed_tuple
-            new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed.to_string)
-        if new_seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(new_seeds)
-        if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(del_seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-

cobweb/schedulers/launcher_api.py

@@ -1,225 +0,0 @@
-import time
-import threading
-
-from cobweb.db import ApiDB
-from cobweb.base import Seed, logger
-from cobweb.constant import DealModel, LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherApi(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._db = ApiDB()
-
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._done_key = "{%s:%s}:done" % (project, task)
-        self._fail_key = "{%s:%s}:fail" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-
-        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
-        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-
-        # self._bf_key = "bloom_%s_%s" % (project, task)
-        # self._bf = BloomFilter(self._bf_key)
-
-        self._heartbeat_start_event = threading.Event()
-        self._redis_queue_empty_event = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def statistics(self, key, count):
-        if not self._task_model and not self._db.exists(key):
-            self._db.setex(key, 86400 * 30, int(count))
-        else:
-            self._db.incrby(key, count)
-
-    def _get_seed(self) -> Seed:
-        """
-        从队列中获取种子（频控）
-        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
-        :return: True -> 种子, False -> None
-        """
-        if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
-                not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
-            expire_time = self._db.ttl(self._speed_control_key)
-            if isinstance(expire_time, int) and expire_time <= -1:
-                self._db.delete(self._speed_control_key)
-            elif isinstance(expire_time, int):
-                logger.info(f"Too fast! Please wait {expire_time} seconds...")
-                time.sleep(expire_time / 2)
-            return None
-        seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        return seed
-
-    @check_pause
-    def _execute_heartbeat(self):
-        if self._heartbeat_start_event.is_set():
-            self._db.setex(self._heartbeat_key, 5)
-        time.sleep(3)
-
-    @check_pause
-    def _reset(self):
-        """
-        检查过期种子，重新添加到redis缓存中
-        """
-        reset_wait_seconds = 30
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            _min = -int(time.time()) + self._seed_reset_seconds \
-                if self.heartbeat else "-inf"
-
-            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self._heartbeat_start_event.set()
-
-            self._db.delete(self._reset_lock_key)
-
-        time.sleep(reset_wait_seconds)
-
-    @check_pause
-    def _scheduler(self):
-        """
-        调度任务，获取redis队列种子，同时添加到doing字典中
-        """
-        if not self._db.zcount(self._todo_key, 0, "(1000"):
-            time.sleep(self._scheduler_wait_seconds)
-        elif self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
-            time.sleep(self._todo_queue_full_wait_seconds)
-        else:
-            members = self._db.members(
-                self._todo_key, int(time.time()),
-                count=self._todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                self.__LAUNCHER_QUEUE__['todo'].push(seed)
-                self.__DOING__[seed.to_string] = seed.params.priority
-
-    @check_pause
-    def _insert(self):
-        """
-        添加新种子到redis队列中
-        """
-        new_seeds = {}
-        del_seeds = set()
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed_tuple:
-                break
-            seed, new_seed = seed_tuple
-            new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed)
-        if new_seeds:
-            self._db.zadd(self._todo_key, new_seeds, nx=True)
-        if del_seeds:
-            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _refresh(self):
-        """
-        刷新doing种子过期时间，防止reset重新消费
-        """
-        if self.__DOING__:
-            refresh_time = int(time.time())
-            seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
-            self._db.zadd(self._todo_key, item=seeds, xx=True)
-        time.sleep(15)
-
-    @check_pause
-    def _delete(self):
-        """
-        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
-        """
-        # seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
-
-        seed_list = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seed_list.append(seed.to_string)
-
-        if seed_list:
-
-            self._db.zrem(self._todo_key, *seed_list)
-            self._remove_doing_seeds(seed_list)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-        wait_scheduler_execute = True
-        check_emtpy_times = 0
-        while not self._stop.is_set():
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-                    wait_scheduler_execute = False
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model and (
-                            not wait_scheduler_execute or
-                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
-                    ):
-                        logger.info("Done! ready to close thread...")
-                        self._stop.set()
-
-                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
-                        logger.info(f"Recovery {self.task} task run！")
-                        self._pause.clear()
-                        self._execute()
-                    else:
-                        logger.info("pause! waiting for resume...")
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    seed_count = self._db.zcard(self._todo_key)
-                    logger.info(f"队列剩余种子数:{seed_count}")
-                    if not seed_count:
-                        logger.info("Done! pause set...")
-                        self._pause.set()
-                    else:
-                        self._pause.clear()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            else:
-                if self._pause.is_set():
-                    self._pause.clear()
-                logger.info(LogTemplate.launcher_pro_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
-                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
-                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-        logger.info("Done! Ready to close thread...")
-