cobweb-launcher 1.0.5__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/utils/oss.py

@@ -1,86 +1,113 @@
-from cobweb import setting
-from requests import Response
-from oss2 import Auth, Bucket, models
-from cobweb.exceptions import oss_db_exception
-from cobweb.base.decorators import decorator_oss_db
-
-
-class OssUtil:
-
-    def __init__(
-        self,
-        bucket=None,
-        endpoint=None,
-        access_key=None,
-        secret_key=None,
-        chunk_size=None,
-        min_upload_size=None,
-    ):
-        self.bucket = bucket or setting.OSS_BUCKET
-        self.endpoint = endpoint or setting.OSS_ENDPOINT
-        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
-        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
-        self._auth = Auth(
-            access_key_id=access_key or setting.OSS_ACCESS_KEY,
-            access_key_secret=secret_key or setting.OSS_SECRET_KEY
-        )
-        self._client = Bucket(
-            auth=self._auth,
-            endpoint=self.endpoint,
-            bucket_name=self.bucket,
-        )
-
-    def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
-
-    def head(self, key: str) -> models.HeadObjectResult:
-        return self._client.head_object(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
-    def init_part(self, key) -> models.InitMultipartUploadResult:
-        """初始化分片上传"""
-        return self._client.init_multipart_upload(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
-    def put(self, key, data) -> models.PutObjectResult:
-        """文件上传"""
-        return self._client.put_object(key, data)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
-    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
-        """分片上传"""
-        return self._client.upload_part(key, upload_id, position, data)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
-    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
-        """合并分片"""
-        headers = None if parts else {"x-oss-complete-all": "yes"}
-        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
-    def append(self, key, position, data) -> models.AppendObjectResult:
-        """追加上传"""
-        return self._client.append_object(key, position, data)
-
-    def iter_data(self, data, chunk_size=None):
-        chunk_size = chunk_size or self.chunk_size
-        if isinstance(data, Response):
-            for part_data in data.iter_content(chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), chunk_size):
-                yield data[i:i + chunk_size]
-
-    def assemble(self, ready_data, data, chunk_size=None):
-        upload_data = b""
-        ready_data = ready_data + data
-        chunk_size = chunk_size or self.chunk_size
-        if len(ready_data) >= chunk_size:
-            upload_data = ready_data[:chunk_size]
-            ready_data = ready_data[chunk_size:]
-        return ready_data, upload_data
-
-    def content_length(self, key: str) -> int:
-        head = self.head(key)
-        return head.content_length
-
+#
+# from cobweb import setting
+# from requests import Response
+# from oss2 import Auth, Bucket, models, PartIterator
+# from cobweb.exceptions import oss_db_exception
+# from cobweb.utils.decorators import decorator_oss_db
+#
+#
+# class OssUtil:
+#
+#     def __init__(
+#         self,
+#         bucket=None,
+#         endpoint=None,
+#         access_key=None,
+#         secret_key=None,
+#         chunk_size=None,
+#         min_upload_size=None,
+#         **kwargs
+#     ):
+#         self.bucket = bucket or setting.OSS_BUCKET
+#         self.endpoint = endpoint or setting.OSS_ENDPOINT
+#         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
+#         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+#
+#         self.failed_count = 0
+#         self._kw = kwargs
+#
+#         self._auth = Auth(
+#             access_key_id=access_key or setting.OSS_ACCESS_KEY,
+#             access_key_secret=secret_key or setting.OSS_SECRET_KEY
+#         )
+#         self._client = Bucket(
+#             auth=self._auth,
+#             endpoint=self.endpoint,
+#             bucket_name=self.bucket,
+#             **self._kw
+#         )
+#
+#     def failed(self):
+#         self.failed_count += 1
+#         if self.failed_count >= 5:
+#             self._client = Bucket(
+#                 auth=self._auth,
+#                 endpoint=self.endpoint,
+#                 bucket_name=self.bucket,
+#                 **self._kw
+#             )
+#
+#     def exists(self, key: str) -> bool:
+#         try:
+#             result = self._client.object_exists(key)
+#             self.failed_count = 0
+#             return result
+#         except Exception as e:
+#             self.failed()
+#             raise e
+#
+#     def head(self, key: str) -> models.HeadObjectResult:
+#         return self._client.head_object(key)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+#     def init_part(self, key) -> models.InitMultipartUploadResult:
+#         """初始化分片上传"""
+#         return self._client.init_multipart_upload(key)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+#     def put(self, key, data) -> models.PutObjectResult:
+#         """文件上传"""
+#         return self._client.put_object(key, data)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+#     def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
+#         """分片上传"""
+#         return self._client.upload_part(key, upload_id, position, data)
+#
+#     def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
+#         """获取分片列表"""
+#         return [part_info for part_info in PartIterator(self._client, key, upload_id)]
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+#     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
+#         """合并分片"""
+#         headers = None if parts else {"x-oss-complete-all": "yes"}
+#         return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+#     def append(self, key, position, data) -> models.AppendObjectResult:
+#         """追加上传"""
+#         return self._client.append_object(key, position, data)
+#
+#     def iter_data(self, data, chunk_size=None):
+#         chunk_size = chunk_size or self.chunk_size
+#         if isinstance(data, Response):
+#             for part_data in data.iter_content(chunk_size):
+#                 yield part_data
+#         if isinstance(data, bytes):
+#             for i in range(0, len(data), chunk_size):
+#                 yield data[i:i + chunk_size]
+#
+#     def assemble(self, ready_data, data, chunk_size=None):
+#         upload_data = b""
+#         ready_data = ready_data + data
+#         chunk_size = chunk_size or self.chunk_size
+#         if len(ready_data) >= chunk_size:
+#             upload_data = ready_data[:chunk_size]
+#             ready_data = ready_data[chunk_size:]
+#         return ready_data, upload_data
+#
+#     def content_length(self, key: str) -> int:
+#         head = self.head(key)
+#         return head.content_length
+#

cobweb/utils/tools.py

@@ -1,5 +1,6 @@
 import re
 import hashlib
+import inspect
 from typing import Union
 from importlib import import_module
 
@@ -10,18 +11,6 @@ def md5(text: Union[str, bytes]) -> str:
     return hashlib.md5(text).hexdigest()
 
 
-def build_path(site, url, file_type):
-    return f"{site}/{md5(url)}.{file_type}"
-
-
-def format_size(content_length: int) -> str:
-    units = ["KB", "MB", "GB", "TB"]
-    for i in range(4):
-        num = content_length / (1024 ** (i + 1))
-        if num < 1024:
-            return f"{round(num, 2)} {units[i]}"
-
-
 def dynamic_load_class(model_info):
     if isinstance(model_info, str):
         if "import" in model_info:
@@ -35,8 +24,7 @@ def dynamic_load_class(model_info):
             model = import_module(model_path)
             class_object = getattr(model, class_name)
         return class_object
+    elif inspect.isclass(model_info):
+        return model_info
     raise TypeError()
 
-
-def download_log_info(item:dict) -> str:
-    return "\n".join([" " * 12 + f"{k.ljust(14)}:    {v}" for k, v in item.items()])

cobweb_launcher-3.2.18.dist-info/METADATA

@@ -0,0 +1,193 @@
+Metadata-Version: 2.1
+Name: cobweb-launcher
+Version: 3.2.18
+Summary: spider_hole
+Home-page: https://github.com/Juannie-PP/cobweb
+Author: Juannie-PP
+Author-email: 2604868278@qq.com
+License: MIT
+Keywords: cobweb-launcher,cobweb,spider
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.19.1
+Requires-Dist: redis>=4.4.4
+Requires-Dist: aliyun-log-python-sdk
+
+# cobweb 
+cobweb是一个基于python的分布式爬虫调度框架，目前支持分布式爬虫，单机爬虫，支持自定义数据库，支持自定义数据存储，支持自定义数据处理等操作。  
+
+cobweb主要由3个模块和一个配置文件组成：Launcher启动器、Crawler采集器、Pipeline存储和setting配置文件。
+1. Launcher启动器：用于启动爬虫任务，控制爬虫任务的执行流程，以及数据存储和数据处理。
+框架提供两种启动器模式：LauncherAir、LauncherPro，分别对应单机爬虫模式和分布式调度模式。
+2. Crawler采集器：用于控制采集流程、数据下载和数据处理。
+框架提供了基础的采集器，用于控制采集流程、数据下载和数据处理，用户也可在创建任务时自定义请求、下载和解析方法，具体看使用方法介绍。
+3. Pipeline存储：用于存储采集到的数据，支持自定义数据存储和数据处理。框架提供了Console和Loghub两种存储方式，用户也可继承Pipeline抽象类自定义存储方式。
+4. setting配置文件：用于配置采集器、存储器、队列长度、采集线程数等参数，框架提供了默认配置，用户也可自定义配置。
+## 安装
+```
+pip3 install --upgrade cobweb-launcher
+```
+## 使用方法介绍
+### 1. 任务创建
+- Launcher任务创建
+```python
+from cobweb import Launcher
+
+# 创建启动器
+app = Launcher(task="test", project="test")
+
+# 设置采集种子
+app.SEEDS = [{
+    "url": "https://www.baidu.com"
+}]
+...
+# 启动任务
+app.start()
+```
+### 2. 自定义配置文件参数
+- 通过自定义setting文件，配置文件导入字符串方式  
+> 默认配置文件：import cobweb.setting  
+> 不推荐！！！目前有bug，随缘使用...
+例如：同级目录下自定义创建了setting.py文件。
+```python
+from cobweb import Launcher
+
+app = Launcher(
+    task="test", 
+    project="test",
+    setting="import setting"
+)
+
+...
+
+app.start()
+```
+- 自定义修改setting中对象值
+```python
+from cobweb import Launcher
+
+# 创建启动器
+app = Launcher(
+    task="test",
+    project="test",
+    REDIS_CONFIG = {
+        "host": ...,
+        "password":...,
+        "port": ...,
+        "db": ...
+    }
+)
+...
+# 启动任务
+app.start()
+```
+### 3. 自定义请求
+`@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
+```python
+from typing import Union
+from cobweb import Launcher
+from cobweb.base import Seed, Request, BaseItem
+
+app = Launcher(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.request
+def request(seed: Seed) -> Union[Request, BaseItem]:
+    # 可自定义headers，代理，构造请求参数等操作
+    proxies = {"http": ..., "https": ...}
+    yield Request(seed.url, seed, ..., proxies=proxies, timeout=15)
+    # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
+
+...
+
+app.start()
+```
+> 默认请求方法  
+> def request(seed: Seed) -> Union[Request, BaseItem]:  
+>     yield Request(seed.url, seed, timeout=5)
+### 4. 自定义下载
+`@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
+```python
+from typing import Union
+from cobweb import Launcher
+from cobweb.base import Request, Response, BaseItem
+
+app = Launcher(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.download
+def download(item: Request) -> Union[BaseItem, Response]:
+    ...
+    response = ...
+    ...
+    yield Response(item.seed, response, ...) # 返回Response对象，进行解析
+    # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
+
+...
+
+app.start()
+```
+> 默认下载方法  
+> def download(item: Request) -> Union[Seed, BaseItem, Response, str]:  
+>     response = item.download()  
+>     yield Response(item.seed, response, **item.to_dict)
+### 5. 自定义解析
+自定义解析需要由一个存储数据类和解析方法组成。存储数据类继承于BaseItem的对象，规定存储表名及字段，
+解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
+```python
+from typing import Union
+from cobweb import Launcher
+from cobweb.base import Seed, Response, BaseItem
+
+class TestItem(BaseItem):
+    __TABLE__ = "test_data" # 表名
+    __FIELDS__ = "field1, field2, field3" # 字段名
+
+app = Launcher(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.parse
+def parse(item: Response) -> Union[Seed, BaseItem]:
+    ...
+    yield TestItem(item.seed, field1=..., field2=..., field3=...)
+    # yield Seed(...) # 构造新种子推送至消费队列
+
+...
+
+app.start()
+```
+> 默认解析方法  
+> def parse(item: Request) -> Union[Seed, BaseItem]:  
+>     upload_item = item.to_dict  
+>     upload_item["text"] = item.response.text  
+>     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+## todo
+- [ ] 队列优化完善，使用queue的机制wait()同步各模块执行？
+- [x] 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- [ ] 去重过滤（布隆过滤器等）
+- [ ] 请求检验
+- [ ] 异常回调
+- [ ] 失败回调
+
+> 未更新流程图！！！
+![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+
+
+

cobweb_launcher-3.2.18.dist-info/RECORD

@@ -0,0 +1,44 @@
+cobweb/__init__.py,sha256=1V2fOFvCncbsPlyzAOo0o6FB9mJfJfaomIVWgNb4hMk,155
+cobweb/constant.py,sha256=u44MFrduzcuITPto9MwPuAGavfDjzi0FKh2looF-9FY,2988
+cobweb/setting.py,sha256=Mte2hQPo1HQUTtOePIfP69GUR7TjeTA2AedaFLndueI,1679
+cobweb/base/__init__.py,sha256=NanSxJr0WsqjqCNOQAlxlkt-vQEsERHYBzacFC057oI,222
+cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
+cobweb/base/item.py,sha256=1bS4U_3vzI2jzSSeoEbLoLT_5CfgLPopWiEYtaahbvw,1674
+cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
+cobweb/base/request.py,sha256=8CrnQQ9q4R6pX_DQmeaytboVxXquuQsYH-kfS8ECjIw,28845
+cobweb/base/response.py,sha256=L3sX2PskV744uz3BJ8xMuAoAfGCeh20w8h0Cnd9vLo0,11377
+cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
+cobweb/base/task_queue.py,sha256=2MqGpHGNmK5B-kqv7z420RWyihzB9zgDHJUiLsmtzOI,6402
+cobweb/base/test.py,sha256=N8MDGb94KQeI4pC5rCc2QdohE9_5AgcOyGqKjbMsOEs,9588
+cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb/crawlers/crawler.py,sha256=ZZVZJ17RWuvzUFGLjqdvyVZpmuq-ynslJwXQzdm_UdQ,709
+cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb/db/api_db.py,sha256=qIhEGB-reKPVFtWPIJYFVK16Us32GBgYjgFjcF-V0GM,3036
+cobweb/db/redis_db.py,sha256=X7dUpW50QcmRPjYlYg7b-fXF_fcjuRRk3DBx2ggetXk,7687
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
+cobweb/launchers/distributor.py,sha256=Ay44iPDNOlrZIW1wgNa4aijRNZsuLLfBz-QlFYXEKDs,5667
+cobweb/launchers/launcher.py,sha256=Shb6o6MAM38d32ybW2gY6qpGmhuiV7jo9TDh0f7rud8,5694
+cobweb/launchers/uploader.py,sha256=QwJOmG7jq2T5sRzrT386zJ0YYNz-hAv0i6GOpoEaRdU,2075
+cobweb/log_dots/__init__.py,sha256=F-3sNXac5uI80j-tNFGPv0lygy00uEI9XfFvK178RC4,55
+cobweb/log_dots/dot.py,sha256=ng6PAtQwR56F-421jJwablAmvMM_n8CR0gBJBJTBMhM,9927
+cobweb/log_dots/loghub_dot.py,sha256=HGbvvVuJqFCOa17-bDeZokhetOmo8J6YpnhL8prRxn4,1741
+cobweb/pipelines/__init__.py,sha256=rtkaaCZ4u1XcxpkDLHztETQjEcLZ_6DXTHjdfcJlyxQ,97
+cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,357
+cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
+cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
+cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
+cobweb/schedulers/scheduler.py,sha256=Of-BjbBh679R6glc12Kc8iugeERCSusP7jolpCc1UMI,1740
+cobweb/schedulers/scheduler_with_redis.py,sha256=iaImkTldr203Xm5kbBWPJ2fLUUe2mgcmUoDUQYfn5iM,7370
+cobweb/utils/__init__.py,sha256=HV0rDW6PVpIVmN_bKquKufli15zuk50tw1ZZejyZ4Pc,173
+cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
+cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
+cobweb/utils/dotting.py,sha256=e0pLEWB8sly1xvXOmJ_uHGQ6Bbw3O9tLcmUBfyNKRmQ,10633
+cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
+cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
+cobweb_launcher-3.2.18.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.2.18.dist-info/METADATA,sha256=bdZ30kO73t3cqozkOyfhr0iEmrDDxPxiqVR9J3oITbE,6115
+cobweb_launcher-3.2.18.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+cobweb_launcher-3.2.18.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.2.18.dist-info/RECORD,,

{cobweb_launcher-1.0.5.dist-info → cobweb_launcher-3.2.18.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cobweb/crawlers/base_crawler.py

@@ -1,121 +0,0 @@
-import threading
-
-from inspect import isgenerator
-from typing import Union, Callable, Mapping
-
-from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
-from cobweb.constant import DealModel, LogTemplate
-from cobweb.utils import download_log_info
-from cobweb import setting
-
-
-class Crawler(threading.Thread):
-
-    def __init__(
-            self,
-            upload_queue: Queue,
-            custom_func: Union[Mapping[str, Callable]],
-            launcher_queue: Union[Mapping[str, Queue]],
-    ):
-        super().__init__()
-
-        self.upload_queue = upload_queue
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.launcher_queue = launcher_queue
-
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
-        self.max_retries = setting.SPIDER_MAX_RETRIES
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        stream = True if setting.DOWNLOAD_MODEL else False
-        return Request(seed.url, seed, stream=stream, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        pass
-
-    def get(self) -> Seed:
-        return self.launcher_queue['todo'].pop()
-
-    def spider(self):
-        while True:
-            seed = self.get()
-
-            if not seed:
-                continue
-
-            elif seed.params.retry >= self.max_retries:
-                self.launcher_queue['done'].push(seed)
-                continue
-
-            item = self.request(seed)
-
-            if isinstance(item, Request):
-
-                download_iterators = self.download(item)
-
-                if not isgenerator(download_iterators):
-                    raise TypeError("download function isn't a generator")
-
-                seed_detail_log_info = download_log_info(seed.to_dict)
-
-                try:
-                    for it in download_iterators:
-                        if isinstance(it, Response):
-                            response_detail_log_info = download_log_info(it.to_dict())
-                            logger.info(LogTemplate.download_info.format(
-                                detail=seed_detail_log_info, retry=item.seed.params.retry,
-                                priority=item.seed.params.priority,
-                                seed_version=item.seed.params.seed_version,
-                                identifier=item.seed.params.identifier,
-                                status=it.response, response=response_detail_log_info
-                            ))
-                            parse_iterators = self.parse(it)
-                            if not isgenerator(parse_iterators):
-                                raise TypeError("parse function isn't a generator")
-                            for upload_item in parse_iterators:
-                                if not isinstance(upload_item, BaseItem):
-                                    raise TypeError("upload_item isn't BaseItem subclass")
-                                self.upload_queue.push(upload_item)
-                        elif isinstance(it, BaseItem):
-                            self.upload_queue.push(it)
-                        elif isinstance(it, Seed):
-                            self.launcher_queue['new'].push(it)
-                        elif isinstance(it, str) and it == DealModel.poll:
-                            self.launcher_queue['todo'].push(item)
-                            break
-                        elif isinstance(it, str) and it == DealModel.done:
-                            self.launcher_queue['done'].push(seed)
-                            break
-                        elif isinstance(it, str) and it == DealModel.fail:
-                            seed.params.identifier = DealModel.fail
-                            self.launcher_queue['done'].push(seed)
-                            break
-                        else:
-                            raise TypeError("yield value type error!")
-
-                except Exception as e:
-                    logger.info(LogTemplate.download_exception.format(
-                        detail=seed_detail_log_info, retry=seed.params.retry,
-                        priority=seed.params.priority, seed_version=seed.params.seed_version,
-                        identifier=seed.params.identifier, exception=e
-                    ))
-                    seed.params.retry += 1
-                    self.launcher_queue['todo'].push(seed)
-
-            elif isinstance(item, BaseItem):
-                self.upload_queue.push(item)
-
-    def run(self):
-        for index in range(self.spider_thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
-