cobweb-launcher 1.2.0__py3-none-any.whl → 1.2.2__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -40,6 +40,7 @@ class Launcher(threading.Thread):
         self.task = task
         self.project = project
 
+        self._app_time = int(time.time())
         self._stop = threading.Event()  # 结束事件
         self._pause = threading.Event()  # 暂停事件
 
@@ -65,6 +66,7 @@ class Launcher(threading.Thread):
         self._Crawler = dynamic_load_class(setting.CRAWLER)
         self._Pipeline = dynamic_load_class(setting.PIPELINE)
 
+        self._before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
         self._scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
         self._todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
         self._new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
@@ -83,7 +85,6 @@ class Launcher(threading.Thread):
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
 
-        # self._upload_queue = Queue()
 
     @property
     def start_seeds(self):
@@ -125,7 +126,7 @@ class Launcher(threading.Thread):
         自定义parse函数, xxxItem为自定义的存储数据类型
         use case:
             from cobweb.base import Request, Response
-            @launcher.download
+            @launcher.parse
             def parse(item: Response) -> BaseItem:
                ...
                yield xxxItem(seed, **kwargs)
@@ -141,7 +142,7 @@ class Launcher(threading.Thread):
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
             threading.Thread(name=func_name, target=getattr(self, func_name)).start()
-            time.sleep(2)
+            time.sleep(1)
 
     def run(self):
         threading.Thread(target=self._execute_heartbeat).start()

cobweb/launchers/launcher_pro.py

@@ -137,6 +137,7 @@ class LauncherPro(Launcher):
                 time.sleep(self._done_queue_wait_seconds)
 
     def _polling(self):
+        wait_scheduler_execute = True
         check_emtpy_times = 0
         while not self._stop.is_set():
             queue_not_empty_count = 0
@@ -145,26 +146,35 @@ class LauncherPro(Launcher):
             for q in self.__LAUNCHER_QUEUE__.values():
                 if q.length != 0:
                     queue_not_empty_count += 1
+                    wait_scheduler_execute = False
 
             if queue_not_empty_count == 0:
                 pooling_wait_seconds = 3
                 if self._pause.is_set():
                     check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
+                    if not self._task_model and (
+                            not wait_scheduler_execute or
+                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
+                    ):
+                        logger.info("Done! ready to close thread...")
                         self._stop.set()
-                elif not self._db.zcount(self._todo_key, _min=0, _max="(1000") and check_emtpy_times > 2:
+
+                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
+                        logger.info(f"Recovery {self.task} task run！")
+                        self._pause.clear()
+                        self._execute()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                elif check_emtpy_times > 2:
                     self.__DOING__ = {}
-                    self._pause.set()
+                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                        self._pause.set()
                 else:
                     logger.info(
                         "check whether the task is complete, "
                         f"reset times {3 - check_emtpy_times}"
                     )
                     check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
             else:
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
@@ -178,36 +188,6 @@ class LauncherPro(Launcher):
                 ))
 
             time.sleep(pooling_wait_seconds)
-            # if self._pause.is_set():
-            #     self._pause.clear()
-            #     self._execute()
-            #
-            # elif queue_not_empty_count == 0:
-            #     pooling_wait_seconds = 5
-            #     check_emtpy_times += 1
-            # else:
-            #     check_emtpy_times = 0
-            #
-            # if not self._db.zcount(self._todo, _min=0, _max="(1000") and check_emtpy_times > 2:
-            #     check_emtpy_times = 0
-            #     self.__DOING__ = {}
-            #     self._pause.set()
-            #
-            # time.sleep(pooling_wait_seconds)
-            #
-            # if not self._pause.is_set():
-            #     logger.info(LogTemplate.launcher_pro_polling.format(
-            #         task=self.task,
-            #         doing_len=len(self.__DOING__.keys()),
-            #         todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-            #         done_len=self.__LAUNCHER_QUEUE__['done'].length,
-            #         redis_seed_count=self._db.zcount(self._todo, "-inf", "+inf"),
-            #         redis_todo_len=self._db.zcount(self._todo, 0, "(1000"),
-            #         redis_doing_len=self._db.zcount(self._todo, "-inf", "(0"),
-            #         upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-            #     ))
-            # elif not self._task_model:
-            #     self._stop.set()
 
         logger.info("Done! Ready to close thread...")
 

cobweb/pipelines/pipeline_console.py

@@ -1,5 +1,3 @@
-import json
-
 from cobweb.base import ConsoleItem, logger
 from cobweb.constant import LogTemplate
 from cobweb.pipelines import Pipeline

cobweb/setting.py

@@ -26,6 +26,9 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
 OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
+# message
+MESSAGE = ""
+
 
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
@@ -35,6 +38,8 @@ PIPELINE = "cobweb.pipelines.pipeline_console.Console"
 
 
 # Launcher 等待时间
+
+BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
 SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
 TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间

cobweb_launcher-1.2.2.dist-info/METADATA

@@ -0,0 +1,204 @@
+Metadata-Version: 2.1
+Name: cobweb-launcher
+Version: 1.2.2
+Summary: spider_hole
+Home-page: https://github.com/Juannie-PP/cobweb
+Author: Juannie-PP
+Author-email: 2604868278@qq.com
+License: MIT
+Keywords: cobweb-launcher, cobweb
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests (>=2.19.1)
+Requires-Dist: oss2 (>=2.18.1)
+Requires-Dist: redis (>=4.4.4)
+Requires-Dist: aliyun-log-python-sdk
+
+# cobweb 
+cobweb是一个基于python的分布式爬虫调度框架，目前支持分布式爬虫，单机爬虫，支持自定义数据库，支持自定义数据存储，支持自定义数据处理等操作。  
+
+cobweb主要由3个模块和一个配置文件组成：Launcher启动器、Crawler采集器、Pipeline存储和setting配置文件。
+1. Launcher启动器：用于启动爬虫任务，控制爬虫任务的执行流程，以及数据存储和数据处理。
+框架提供两种启动器模式：LauncherAir、LauncherPro，分别对应单机爬虫模式和分布式调度模式。
+2. Crawler采集器：用于控制采集流程、数据下载和数据处理。
+框架提供了基础的采集器，用于控制采集流程、数据下载和数据处理，用户也可在创建任务时自定义请求、下载和解析方法，具体看使用方法介绍。
+3. Pipeline存储：用于存储采集到的数据，支持自定义数据存储和数据处理。框架提供了Console和Loghub两种存储方式，用户也可继承Pipeline抽象类自定义存储方式。
+4. setting配置文件：用于配置采集器、存储器、队列长度、采集线程数等参数，框架提供了默认配置，用户也可自定义配置。
+## 安装
+```
+pip3 install --upgrade cobweb-launcher
+```
+## 使用方法介绍
+### 1. 任务创建
+- LauncherAir任务创建
+```python
+from cobweb import LauncherAir
+
+# 创建启动器
+app = LauncherAir(task="test", project="test")
+
+# 设置采集种子
+app.SEEDS = [{
+    "url": "https://www.baidu.com"
+}]
+...
+# 启动任务
+app.start()
+```
+- LauncherPro任务创建  
+LauncherPro依赖redis实现分布式调度，使用LauncherPro启动器需要完成环境变量的配置或自定义setting文件中的redis配置，如何配置查看`2. 自定义配置文件参数`
+```python
+from cobweb import LauncherPro
+
+# 创建启动器
+app = LauncherPro(
+    task="test",
+    project="test"
+)
+...
+# 启动任务
+app.start()
+```
+### 2. 自定义配置文件参数
+- 通过自定义setting文件，配置文件导入字符串方式  
+> 默认配置文件：import cobweb.setting  
+> 不推荐！！！目前有bug，随缘使用...
+例如：同级目录下自定义创建了setting.py文件。
+```python
+from cobweb import LauncherAir
+
+app = LauncherAir(
+    task="test", 
+    project="test",
+    setting="import setting"
+)
+
+...
+
+app.start()
+```
+- 自定义修改setting中对象值
+```python
+from cobweb import LauncherPro
+
+# 创建启动器
+app = LauncherPro(
+    task="test",
+    project="test",
+    REDIS_CONFIG = {
+        "host": ...,
+        "password":...,
+        "port": ...,
+        "db": ...
+    }
+)
+...
+# 启动任务
+app.start()
+```
+### 3. 自定义请求
+`@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
+```python
+from typing import Union
+from cobweb import LauncherAir
+from cobweb.base import Seed, Request, BaseItem
+
+app = LauncherAir(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.request
+def request(seed: Seed) -> Union[Request, BaseItem]:
+    # 可自定义headers，代理，构造请求参数等操作
+    proxies = {"http": ..., "https": ...}
+    yield Request(seed.url, seed, ..., proxies=proxies, timeout=15)
+    # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
+    
+...
+
+app.start()
+```
+> 默认请求方法  
+> def request(seed: Seed) -> Union[Request, BaseItem]:  
+>     yield Request(seed.url, seed, timeout=5)
+### 4. 自定义下载
+`@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
+```python
+from typing import Union
+from cobweb import LauncherAir
+from cobweb.base import Request, Response, BaseItem
+
+app = LauncherAir(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.download
+def download(item: Request) -> Union[BaseItem, Response]:
+    ...
+    response = ...
+    ...
+    yield Response(item.seed, response, ...) # 返回Response对象，进行解析
+    # yield xxxItem(seed, ...) # 跳过请求和解析直接进入数据存储流程
+    
+...
+
+app.start()
+```
+> 默认下载方法  
+> def download(item: Request) -> Union[Seed, BaseItem, Response, str]:  
+>     response = item.download()  
+>     yield Response(item.seed, response, **item.to_dict)
+### 5. 自定义解析
+自定义解析需要由一个存储数据类和解析方法组成。存储数据类继承于BaseItem的对象，规定存储表名及字段，
+解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
+```python
+from typing import Union
+from cobweb import LauncherAir
+from cobweb.base import Seed, Response, BaseItem
+
+class TestItem(BaseItem):
+    __TABLE__ = "test_data" # 表名
+    __FIELDS__ = "field1, field2, field3" # 字段名
+
+app = LauncherAir(
+    task="test", 
+    project="test"
+)
+
+...
+
+@app.parse
+def parse(item: Response) -> Union[Seed, BaseItem]:
+    ...
+    yield TestItem(item.seed, field1=..., field2=..., field3=...)
+    # yield Seed(...) # 构造新种子推送至消费队列
+    
+...
+
+app.start()
+```
+> 默认解析方法  
+> def parse(item: Request) -> Union[Seed, BaseItem]:  
+>     upload_item = item.to_dict  
+>     upload_item["text"] = item.response.text  
+>     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+## need deal
+- 队列优化完善，使用queue的机制wait()同步各模块执行？
+- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- 去重过滤（布隆过滤器等）
+- 单机防丢失
+- excel、mysql、redis数据完善
+
+> 未更新流程图！！！
+![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+

{cobweb_launcher-1.2.0.dist-info → cobweb_launcher-1.2.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=zOO1cA_zQd4Q0CzY_tdSfdo-10L4QIVpm4382wbP5BQ,1906
+cobweb/setting.py,sha256=_t3LMSpxUNR4dVD9Tox22W9omHPvjeWABFzpgkEOoH0,2016
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -18,20 +18,20 @@ cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=zef9gQ0P_4lD3mButsbIXPr15pqZDyr9wlELCD3bsZs,5382
+cobweb/launchers/launcher.py,sha256=Mepg-hv9YL5_VnY_Mwqxzd8ZRGyK8gxsl_5B_Ibz0Uc,5466
 cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
-cobweb/launchers/launcher_pro.py,sha256=IrrevNxmH39GySwE36RGBhtXatEGT3fffDFoJuzJbaM,8584
+cobweb/launchers/launcher_pro.py,sha256=ihsEcdP8iOrJhHzQTYQow-8pIYK3CCn-iwwI3v9UzMw,7727
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
 cobweb/pipelines/pipeline.py,sha256=29O7CzASDv52NJl_1KWlKYxNd79HiaQAm0oO5K9PQzw,1650
-cobweb/pipelines/pipeline_console.py,sha256=2ur-5nZvCSFBYFMIdePcK2mHp1ktJTBe7vRyH-kaF94,767
+cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.0.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.0.dist-info/METADATA,sha256=wl8ODS1cutZUvE4HmsvsH_42atwhdElRcD5d6rsgSBk,1245
-cobweb_launcher-1.2.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.0.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.0.dist-info/RECORD,,
+cobweb_launcher-1.2.2.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.2.dist-info/METADATA,sha256=nMfaL4ItGVnr9tHtxNnBEGud1HI_ZRbIGqmfL4TnV7o,6489
+cobweb_launcher-1.2.2.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.2.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.2.dist-info/RECORD,,

cobweb_launcher-1.2.0.dist-info/METADATA

@@ -1,48 +0,0 @@
-Metadata-Version: 2.1
-Name: cobweb-launcher
-Version: 1.2.0
-Summary: spider_hole
-Home-page: https://github.com/Juannie-PP/cobweb
-Author: Juannie-PP
-Author-email: 2604868278@qq.com
-License: MIT
-Keywords: cobweb-launcher, cobweb
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: requests (>=2.19.1)
-Requires-Dist: oss2 (>=2.18.1)
-Requires-Dist: redis (>=4.4.4)
-Requires-Dist: aliyun-log-python-sdk
-
-# cobweb
-
-> 通用爬虫框架： 1.单机模式采集框架；2.分布式采集框架
-> 
->  5部分
->  
-> 1. starter -- 启动器
-> 
-> 2. scheduler -- 调度器
-> 
-> 3. distributor -- 分发器
-> 
-> 4. storer -- 存储器
-> 
-> 5. utils -- 工具函数
-> 
-
-need deal
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 防丢失（单机模式可以通过日志文件进行检查种子）
-- 自定义数据库的功能
-- excel、mysql、redis数据完善
-
-
-![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
-
-