cobweb-launcher 1.2.25__py3-none-any.whl → 3.2.20__py3-none-any.whl

{cobweb_launcher-1.2.25.dist-info → cobweb_launcher-3.2.20.dist-info}/METADATA

@@ -1,22 +1,21 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.25
+Version: 3.2.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
 Author-email: 2604868278@qq.com
 License: MIT
-Keywords: cobweb-launcher, cobweb
+Keywords: cobweb-launcher,cobweb,spider
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: requests (>=2.19.1)
-Requires-Dist: oss2 (>=2.18.1)
-Requires-Dist: redis (>=4.4.4)
+Requires-Dist: requests>=2.19.1
+Requires-Dist: redis>=4.4.4
 Requires-Dist: aliyun-log-python-sdk
-Requires-Dist: mmh3
 
 # cobweb 
 cobweb是一个基于python的分布式爬虫调度框架，目前支持分布式爬虫，单机爬虫，支持自定义数据库，支持自定义数据存储，支持自定义数据处理等操作。  
@@ -34,12 +33,12 @@ pip3 install --upgrade cobweb-launcher
 ```
 ## 使用方法介绍
 ### 1. 任务创建
-- LauncherAir任务创建
+- Launcher任务创建
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherAir(task="test", project="test")
+app = Launcher(task="test", project="test")
 
 # 设置采集种子
 app.SEEDS = [{
@@ -49,29 +48,15 @@ app.SEEDS = [{
 # 启动任务
 app.start()
 ```
-- LauncherPro任务创建  
-LauncherPro依赖redis实现分布式调度，使用LauncherPro启动器需要完成环境变量的配置或自定义setting文件中的redis配置，如何配置查看`2. 自定义配置文件参数`
-```python
-from cobweb import LauncherPro
-
-# 创建启动器
-app = LauncherPro(
-    task="test",
-    project="test"
-)
-...
-# 启动任务
-app.start()
-```
 ### 2. 自定义配置文件参数
 - 通过自定义setting文件，配置文件导入字符串方式  
 > 默认配置文件：import cobweb.setting  
 > 不推荐！！！目前有bug，随缘使用...
 例如：同级目录下自定义创建了setting.py文件。
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test",
     setting="import setting"
@@ -83,10 +68,10 @@ app.start()
 ```
 - 自定义修改setting中对象值
 ```python
-from cobweb import LauncherPro
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherPro(
+app = Launcher(
     task="test",
     project="test",
     REDIS_CONFIG = {
@@ -104,10 +89,10 @@ app.start()
 `@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Request, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -120,7 +105,7 @@ def request(seed: Seed) -> Union[Request, BaseItem]:
     proxies = {"http": ..., "https": ...}
     yield Request(seed.url, seed, ..., proxies=proxies, timeout=15)
     # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
-    
+
 ...
 
 app.start()
@@ -132,10 +117,10 @@ app.start()
 `@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Request, Response, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -149,7 +134,7 @@ def download(item: Request) -> Union[BaseItem, Response]:
     ...
     yield Response(item.seed, response, ...) # 返回Response对象，进行解析
     # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
-    
+
 ...
 
 app.start()
@@ -163,14 +148,14 @@ app.start()
 解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Response, BaseItem
 
 class TestItem(BaseItem):
     __TABLE__ = "test_data" # 表名
     __FIELDS__ = "field1, field2, field3" # 字段名
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -182,7 +167,7 @@ def parse(item: Response) -> Union[Seed, BaseItem]:
     ...
     yield TestItem(item.seed, field1=..., field2=..., field3=...)
     # yield Seed(...) # 构造新种子推送至消费队列
-    
+
 ...
 
 app.start()
@@ -192,14 +177,17 @@ app.start()
 >     upload_item = item.to_dict  
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-## need deal
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 单机防丢失
-- excel、mysql、redis数据完善
+## todo
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
 
 
+
+

cobweb_launcher-3.2.20.dist-info/RECORD

@@ -0,0 +1,44 @@
+cobweb/__init__.py,sha256=1V2fOFvCncbsPlyzAOo0o6FB9mJfJfaomIVWgNb4hMk,155
+cobweb/constant.py,sha256=u44MFrduzcuITPto9MwPuAGavfDjzi0FKh2looF-9FY,2988
+cobweb/setting.py,sha256=Mte2hQPo1HQUTtOePIfP69GUR7TjeTA2AedaFLndueI,1679
+cobweb/base/__init__.py,sha256=NanSxJr0WsqjqCNOQAlxlkt-vQEsERHYBzacFC057oI,222
+cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
+cobweb/base/item.py,sha256=1bS4U_3vzI2jzSSeoEbLoLT_5CfgLPopWiEYtaahbvw,1674
+cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
+cobweb/base/request.py,sha256=8CrnQQ9q4R6pX_DQmeaytboVxXquuQsYH-kfS8ECjIw,28845
+cobweb/base/response.py,sha256=L3sX2PskV744uz3BJ8xMuAoAfGCeh20w8h0Cnd9vLo0,11377
+cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
+cobweb/base/task_queue.py,sha256=2MqGpHGNmK5B-kqv7z420RWyihzB9zgDHJUiLsmtzOI,6402
+cobweb/base/test.py,sha256=N8MDGb94KQeI4pC5rCc2QdohE9_5AgcOyGqKjbMsOEs,9588
+cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb/crawlers/crawler.py,sha256=ZZVZJ17RWuvzUFGLjqdvyVZpmuq-ynslJwXQzdm_UdQ,709
+cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb/db/api_db.py,sha256=qIhEGB-reKPVFtWPIJYFVK16Us32GBgYjgFjcF-V0GM,3036
+cobweb/db/redis_db.py,sha256=X7dUpW50QcmRPjYlYg7b-fXF_fcjuRRk3DBx2ggetXk,7687
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
+cobweb/launchers/distributor.py,sha256=1ooOFqWqZLCn7zIDoraDDGJm9l7aAWzygz-UNUSqxpI,5662
+cobweb/launchers/launcher.py,sha256=Shb6o6MAM38d32ybW2gY6qpGmhuiV7jo9TDh0f7rud8,5694
+cobweb/launchers/uploader.py,sha256=QwJOmG7jq2T5sRzrT386zJ0YYNz-hAv0i6GOpoEaRdU,2075
+cobweb/log_dots/__init__.py,sha256=F-3sNXac5uI80j-tNFGPv0lygy00uEI9XfFvK178RC4,55
+cobweb/log_dots/dot.py,sha256=ng6PAtQwR56F-421jJwablAmvMM_n8CR0gBJBJTBMhM,9927
+cobweb/log_dots/loghub_dot.py,sha256=HGbvvVuJqFCOa17-bDeZokhetOmo8J6YpnhL8prRxn4,1741
+cobweb/pipelines/__init__.py,sha256=rtkaaCZ4u1XcxpkDLHztETQjEcLZ_6DXTHjdfcJlyxQ,97
+cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,357
+cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
+cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
+cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
+cobweb/schedulers/scheduler.py,sha256=Of-BjbBh679R6glc12Kc8iugeERCSusP7jolpCc1UMI,1740
+cobweb/schedulers/scheduler_with_redis.py,sha256=yz6anTM3eNOQRP5WtlY1Kx6KaVgKgyR_FwvXtlydGqQ,7403
+cobweb/utils/__init__.py,sha256=HV0rDW6PVpIVmN_bKquKufli15zuk50tw1ZZejyZ4Pc,173
+cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
+cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
+cobweb/utils/dotting.py,sha256=e0pLEWB8sly1xvXOmJ_uHGQ6Bbw3O9tLcmUBfyNKRmQ,10633
+cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
+cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
+cobweb_launcher-3.2.20.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.2.20.dist-info/METADATA,sha256=GrOWnfJGxsO3LPt3_xWpb2ySDzRrupBYngzOpZ7NSPs,6115
+cobweb_launcher-3.2.20.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+cobweb_launcher-3.2.20.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.2.20.dist-info/RECORD,,

{cobweb_launcher-1.2.25.dist-info → cobweb_launcher-3.2.20.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cobweb/crawlers/base_crawler.py

@@ -1,144 +0,0 @@
-import threading
-import time
-import traceback
-
-from inspect import isgenerator
-from typing import Union, Callable, Mapping
-
-from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
-from cobweb.constant import DealModel, LogTemplate
-from cobweb.utils import download_log_info
-from cobweb import setting
-
-
-class Crawler(threading.Thread):
-
-    def __init__(
-            self,
-            upload_queue: Queue,
-            custom_func: Union[Mapping[str, Callable]],
-            launcher_queue: Union[Mapping[str, Queue]],
-    ):
-        super().__init__()
-
-        self.upload_queue = upload_queue
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.launcher_queue = launcher_queue
-
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
-        self.max_retries = setting.SPIDER_MAX_RETRIES
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        stream = True if setting.DOWNLOAD_MODEL else False
-        yield Request(seed.url, seed, stream=stream, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response, **item.to_dict)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        pass
-
-    def get_seed(self) -> Seed:
-        return self.launcher_queue['todo'].pop()
-
-    def distribute(self, item, seed):
-        if isinstance(item, BaseItem):
-            self.upload_queue.push(item)
-        elif isinstance(item, Seed):
-            self.launcher_queue['new'].push(item)
-        elif isinstance(item, str) and item == DealModel.poll:
-            self.launcher_queue['todo'].push(seed)
-        elif isinstance(item, str) and item == DealModel.done:
-            self.launcher_queue['done'].push(seed)
-        elif isinstance(item, str) and item == DealModel.fail:
-            seed.params.seed_status = DealModel.fail
-            self.launcher_queue['done'].push(seed)
-        else:
-            raise TypeError("yield value type error!")
-
-    def spider(self):
-        while True:
-            seed = self.get_seed()
-
-            if not seed:
-                continue
-
-            elif seed.params.retry >= self.max_retries:
-                seed.params.seed_status = DealModel.fail
-                self.launcher_queue['done'].push(seed)
-                continue
-
-            seed_detail_log_info = download_log_info(seed.to_dict)
-
-            try:
-                request_iterators = self.request(seed)
-
-                if not isgenerator(request_iterators):
-                    raise TypeError("request function isn't a generator!")
-
-                iterator_status = False
-
-                for request_item in request_iterators:
-
-                    iterator_status = True
-
-                    if isinstance(request_item, Request):
-                        iterator_status = False
-                        download_iterators = self.download(request_item)
-                        if not isgenerator(download_iterators):
-                            raise TypeError("download function isn't a generator")
-
-                        for download_item in download_iterators:
-                            iterator_status = True
-                            if isinstance(download_item, Response):
-                                iterator_status = False
-                                logger.info(LogTemplate.download_info.format(
-                                    detail=seed_detail_log_info,
-                                    retry=seed.params.retry,
-                                    priority=seed.params.priority,
-                                    seed_version=seed.params.seed_version,
-                                    identifier=seed.identifier or "",
-                                    status=download_item.response,
-                                    response=download_log_info(download_item.to_dict)
-                                ))
-                                parse_iterators = self.parse(download_item)
-                                if not isgenerator(parse_iterators):
-                                    raise TypeError("parse function isn't a generator")
-                                for parse_item in parse_iterators:
-                                    iterator_status = True
-                                    if isinstance(parse_item, Response):
-                                        raise TypeError("upload_item can't be a Response instance")
-                                    self.distribute(parse_item, seed)
-                            else:
-                                self.distribute(download_item, seed)
-                    else:
-                        self.distribute(request_item, seed)
-
-                if not iterator_status:
-                    raise ValueError("request/download/parse function yield value error!")
-
-            except Exception as e:
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                self.launcher_queue['todo'].push(seed)
-            finally:
-                time.sleep(0.1)
-
-    def run(self):
-        for index in range(self.spider_thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
-

cobweb/crawlers/file_crawler.py

@@ -1,98 +0,0 @@
-import os
-from typing import Union
-from cobweb import setting
-from cobweb.utils import OssUtil
-from cobweb.crawlers import Crawler
-from cobweb.base import Seed, BaseItem, Request, Response
-from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
-
-
-oss_util = OssUtil(is_path_style=bool(int(os.getenv("PRIVATE_LINK", 0))))
-
-
-class FileCrawlerAir(Crawler):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        seed_dict["bucket_name"] = oss_util.bucket
-        try:
-            seed_dict["oss_path"] = key = item.seed.oss_path or getattr(item, "oss_path")
-
-            if oss_util.exists(key):
-                seed_dict["data_size"] = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", **seed_dict)
-
-            else:
-                seed_dict.setdefault("end", "")
-                seed_dict.setdefault("start", 0)
-
-                if seed_dict["end"] or seed_dict["start"]:
-                    start, end = seed_dict["start"], seed_dict["end"]
-                    item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-                if not item.seed.identifier:
-                    content = b""
-                    chunk_size = oss_util.chunk_size
-                    min_upload_size = oss_util.min_upload_size
-                    seed_dict.setdefault("position", 1)
-
-                    response = item.download()
-
-                    content_type = response.headers.get("content-type", "").split(";")[0]
-                    seed_dict["data_size"] = content_length = int(response.headers.get("content-length", 0))
-
-                    if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                        """过滤响应文件类型"""
-                        response.close()
-                        seed_dict["filter"] = True
-                        seed_dict["msg"] = f"response content type is {content_type}"
-                        yield Response(item.seed, response, **seed_dict)
-
-                    elif seed_dict['position'] == 1 and min_upload_size >= content_length > 0:
-                        """过小文件标识返回"""
-                        response.close()
-                        seed_dict["filter"] = True
-                        seed_dict["msg"] = "file size is too small"
-                        yield Response(item.seed, response, **seed_dict)
-
-                    elif seed_dict['position'] == 1 and chunk_size > content_length > min_upload_size:
-                        """小文件直接下载"""
-                        for part_data in response.iter_content(chunk_size):
-                            content += part_data
-                        response.close()
-                        oss_util.put(key, content)
-                        yield Response(item.seed, response, **seed_dict)
-
-                    else:
-                        """中大文件同步分片下载"""
-                        seed_dict.setdefault("upload_id", oss_util.init_part(key).upload_id)
-
-                        for part_data in response.iter_content(chunk_size):
-                            content += part_data
-                            if len(content) >= chunk_size:
-                                upload_data = content[:chunk_size]
-                                content = content[chunk_size:]
-                                oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
-                                seed_dict['start'] += len(upload_data)
-                                seed_dict['position'] += 1
-
-                        response.close()
-
-                        if content:
-                            oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
-                        oss_util.merge(key, seed_dict["upload_id"])
-                        seed_dict["data_size"] = oss_util.head(key).content_length
-                        yield Response(item.seed, response, **seed_dict)
-
-                elif item.seed.identifier == "merge":
-                    oss_util.merge(key, seed_dict["upload_id"])
-                    seed_dict["data_size"] = oss_util.head(key).content_length
-                    yield Response(item.seed, "merge", **seed_dict)
-
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")
-
-

cobweb/launchers/launcher_air.py

@@ -1,88 +0,0 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        seeds = {}
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
-                break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-