cobweb-launcher 1.2.6__py3-none-any.whl → 1.2.8__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -2,12 +2,25 @@ import time
 import inspect
 import threading
 import importlib
+from functools import wraps
 
 from cobweb import setting
-from cobweb.base import Seed, Queue
+from cobweb.base import Seed, Queue, logger
 from cobweb.utils.tools import dynamic_load_class
 
 
+def check_pause(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self._pause.is_set():
+            try:
+                func(*args, **kwargs)
+            except Exception as e:
+                logger.info(f"{func.__name__}: " + str(e))
+
+    return wrapper
+
+
 class Launcher(threading.Thread):
 
     SEEDS = []
@@ -85,11 +98,6 @@ class Launcher(threading.Thread):
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
 
-
-    @property
-    def start_seeds(self):
-        return [Seed(seed) for seed in self.SEEDS]
-
     @property
     def request(self):
         """
@@ -135,9 +143,15 @@ class Launcher(threading.Thread):
             self.__CUSTOM_FUNC__["parse"] = func
         return decorator
 
+    def start_seeds(self):
+        seeds = [Seed(seed) for seed in self.SEEDS]
+        self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+        return seeds
+
     def _remove_doing_seeds(self, seeds):
         for seed in seeds:
             self.__DOING__.pop(seed, None)
+        logger.info("remove %s seeds from __DOING__" % len(seeds))
 
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
@@ -147,6 +161,8 @@ class Launcher(threading.Thread):
     def run(self):
         threading.Thread(target=self._execute_heartbeat).start()
 
+        self.start_seeds()
+
         self._Crawler(
             stop=self._stop, pause=self._pause,
             launcher_queue=self.__LAUNCHER_QUEUE__,

cobweb/launchers/launcher_air.py

@@ -1,46 +1,46 @@
 import time
 
-from cobweb.constant import LogTemplate
 from cobweb.base import logger
-from .launcher import Launcher
+from cobweb.constant import LogTemplate
+from .launcher import Launcher, check_pause
 
 
 class LauncherAir(Launcher):
 
-    def _scheduler(self):
-        if self.start_seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
+    # def _scheduler(self):
+    #     if self.start_seeds:
+    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
 
+    @check_pause
     def _insert(self):
-        while not self._pause.is_set():
-            seeds = {}
-            status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-            for _ in range(self._new_queue_max_size):
-                seed = self.__LAUNCHER_QUEUE__['new'].pop()
-                if not seed:
-                    break
-                seeds[seed.to_string] = seed.params.priority
-            if seeds:
-                self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-            if status:
-                time.sleep(self._new_queue_wait_seconds)
-
+        seeds = {}
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed:
+                break
+            seeds[seed.to_string] = seed.params.priority
+        if seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
+
+    @check_pause
     def _delete(self):
-        while not self._pause.is_set():
-            seeds = []
-            status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+        seeds = []
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
 
-            for _ in range(self._done_queue_max_size):
-                seed = self.__LAUNCHER_QUEUE__['done'].pop()
-                if not seed:
-                    break
-                seeds.append(seed.to_string)
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            seeds.append(seed.to_string)
 
-            if seeds:
-                self._remove_doing_seeds(seeds)
+        if seeds:
+            self._remove_doing_seeds(seeds)
 
-            if status:
-                time.sleep(self._done_queue_wait_seconds)
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
 
     def _polling(self):
 

cobweb/launchers/launcher_pro.py

@@ -4,7 +4,7 @@ import threading
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
 from cobweb.constant import DealModel, LogTemplate
-from .launcher import Launcher
+from .launcher import Launcher, check_pause
 
 
 class LauncherPro(Launcher):
@@ -33,44 +33,41 @@ class LauncherPro(Launcher):
         else:
             self._db._client.incrby(key, count)
 
+    @check_pause
     def _execute_heartbeat(self):
-        while not self._stop.is_set():
-            if self._heartbeat_start_event.is_set():
-                self._db.setex(self._heartbeat_key, 5)
-            time.sleep(3)
+        if self._heartbeat_start_event.is_set():
+            self._db.setex(self._heartbeat_key, 5)
+        time.sleep(3)
 
+    @check_pause
     def _reset(self):
         """
         检查过期种子，重新添加到redis缓存中
         """
-        while not self._pause.is_set():
-            reset_wait_seconds = 30
-            if self._db.lock(self._reset_lock_key, t=120):
+        reset_wait_seconds = 30
+        if self._db.lock(self._reset_lock_key, t=120):
 
-                _min = -int(time.time()) + self._seed_reset_seconds \
-                    if self.heartbeat else "-inf"
+            _min = -int(time.time()) + self._seed_reset_seconds \
+                if self.heartbeat else "-inf"
 
-                self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-                self._db.delete(self._reset_lock_key)
+            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
+            self._db.delete(self._reset_lock_key)
 
-                if not self.heartbeat:
-                    self._heartbeat_start_event.set()
+            if not self.heartbeat:
+                self._heartbeat_start_event.set()
 
-            time.sleep(reset_wait_seconds)
+        time.sleep(reset_wait_seconds)
 
+    @check_pause
     def _scheduler(self):
         """
         调度任务，获取redis队列种子，同时添加到doing字典中
         """
-        if self.start_seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-        while not self._pause.is_set():
-            if not self._db.zcount(self._todo_key, 0, "(1000"):
-                time.sleep(self._scheduler_wait_seconds)
-                continue
-            if self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
-                time.sleep(self._todo_queue_full_wait_seconds)
-                continue
+        if not self._db.zcount(self._todo_key, 0, "(1000"):
+            time.sleep(self._scheduler_wait_seconds)
+        elif self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
+            time.sleep(self._todo_queue_full_wait_seconds)
+        else:
             members = self._db.members(
                 self._todo_key, int(time.time()),
                 count=self._todo_queue_size,
@@ -81,67 +78,66 @@ class LauncherPro(Launcher):
                 self.__LAUNCHER_QUEUE__['todo'].push(seed)
                 self.__DOING__[seed.to_string] = seed.params.priority
 
+    @check_pause
     def _insert(self):
         """
         添加新种子到redis队列中
         """
-        while not self._pause.is_set():
-            seeds = {}
-            status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-            for _ in range(self._new_queue_max_size):
-                seed = self.__LAUNCHER_QUEUE__['new'].pop()
-                if not seed:
-                    break
+        seeds = {}
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['new'].pop()
+            if seed:
                 seeds[seed.to_string] = seed.params.priority
-            if seeds:
-                self._db.zadd(self._todo_key, seeds, nx=True)
-            if status:
-                time.sleep(self._new_queue_wait_seconds)
+        if seeds:
+            self._db.zadd(self._todo_key, seeds, nx=True)
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
 
+    @check_pause
     def _refresh(self):
         """
         刷新doing种子过期时间，防止reset重新消费
         """
-        while not self._pause.is_set():
-            if self.__DOING__:
-                refresh_time = int(time.time())
-                seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
-                self._db.zadd(self._todo_key, item=seeds, xx=True)
-            time.sleep(15)
+        if self.__DOING__:
+            refresh_time = int(time.time())
+            seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
+            self._db.zadd(self._todo_key, item=seeds, xx=True)
+        time.sleep(15)
 
+    @check_pause
     def _delete(self):
         """
         删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
         """
-        while not self._pause.is_set():
-            seeds, s_seeds, f_seeds = [], [], []
-            status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-            for _ in range(self._done_queue_max_size):
-                seed = self.__LAUNCHER_QUEUE__['done'].pop()
-                if not seed:
-                    break
-                if seed.params.seed_status == DealModel.fail:
-                    f_seeds.append(seed.to_string)
-                elif self._done_model == 1:
-                    s_seeds.append(seed.to_string)
-                else:
-                    seeds.append(seed.to_string)
-            if seeds:
-                count = self._db.zrem(self._todo_key, *seeds)
-                self.statistics(self._statistics_done_key, count)
-                self._remove_doing_seeds(seeds)
-            if s_seeds:
-                count = self._db.done([self._todo_key, self._done_key], *s_seeds)
-                self.statistics(self._statistics_done_key, count)
-                self._remove_doing_seeds(s_seeds)
-            if f_seeds:
-                count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
-                self.statistics(self._statistics_fail_key, count)
-                self._remove_doing_seeds(f_seeds)
-
-            if status:
-                time.sleep(self._done_queue_wait_seconds)
+        seeds, s_seeds, f_seeds = [], [], []
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            if seed.params.seed_status == DealModel.fail:
+                f_seeds.append(seed.to_string)
+            elif self._done_model == 1:
+                s_seeds.append(seed.to_string)
+            else:
+                seeds.append(seed.to_string)
+        if seeds:
+            count = self._db.zrem(self._todo_key, *seeds)
+            self.statistics(self._statistics_done_key, count)
+            self._remove_doing_seeds(seeds)
+        if s_seeds:
+            count = self._db.done([self._todo_key, self._done_key], *s_seeds)
+            self.statistics(self._statistics_done_key, count)
+            self._remove_doing_seeds(s_seeds)
+        if f_seeds:
+            count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
+            self.statistics(self._statistics_fail_key, count)
+            self._remove_doing_seeds(f_seeds)
+
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
 
     def _polling(self):
         wait_scheduler_execute = True

cobweb/setting.py

@@ -26,9 +26,6 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
 OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
-# message
-MESSAGE = ""
-
 
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
@@ -43,7 +40,7 @@ BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次
 SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
 TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
-DONE_QUEUE_WAIT_SECONDS = 15   # done队列等待时间
+DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
 SEED_RESET_SECONDS = 30   # 种子重制时间
 

{cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.6
+Version: 1.2.8
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.6.dist-info → cobweb_launcher-1.2.8.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=bKM2J-yjGHeQn1Yf-yZ3cay_FTCPaokgOYSC6GPWwM4,2015
+cobweb/setting.py,sha256=Wev0clo4ZETI5cRvBnzTWnJWo0Nowv_uvNCqlzYPSiE,1990
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -18,9 +18,9 @@ cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=0YhJtmN048MgTZ7SAUWvaKJEPXrNx0p4dkoTopsyjzc,5466
-cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
-cobweb/launchers/launcher_pro.py,sha256=qCmobbi1jhI09cVRjaYGxyTHwBSvARiHxF1p37b5iaE,8107
+cobweb/launchers/launcher.py,sha256=H0MEL8mTcNgNbuZBKlHH0GlQ3IIFvbJkHaG42mZ2vW8,5947
+cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
+cobweb/launchers/launcher_pro.py,sha256=IpckN0Xl5NAPVjuq-pWbCx0RerDmYG9gZ182EDURixo,7599
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
@@ -30,8 +30,8 @@ cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXt
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.6.dist-info/METADATA,sha256=q_ihYp0LYgZRMdvzJ2gmTYG0HgeoOAnRjOEj5FAUp34,6489
-cobweb_launcher-1.2.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.6.dist-info/RECORD,,
+cobweb_launcher-1.2.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.8.dist-info/METADATA,sha256=VCstetX4haZKrB1Y5KWd9D9WjrmxnZORL7ZRX3WiADA,6489
+cobweb_launcher-1.2.8.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.8.dist-info/RECORD,,