cobweb-launcher 1.2.5__tar.gz → 1.2.7__tar.gz

{cobweb-launcher-1.2.5/cobweb_launcher.egg-info → cobweb-launcher-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.5
+Version: 1.2.7
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.5 → cobweb-launcher-1.2.7}/cobweb/crawlers/crawler.py

@@ -86,7 +86,7 @@ class Crawler(threading.Thread):
                 time.sleep(1)
                 continue
 
-            elif seed.params.retry >= self.max_retries:
+            elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
                 self._done.push(seed)
                 continue

{cobweb-launcher-1.2.5 → cobweb-launcher-1.2.7}/cobweb/launchers/launcher.py

@@ -4,7 +4,7 @@ import threading
 import importlib
 
 from cobweb import setting
-from cobweb.base import Seed, Queue
+from cobweb.base import Seed, Queue, logger
 from cobweb.utils.tools import dynamic_load_class
 
 
@@ -79,8 +79,8 @@ class Launcher(threading.Thread):
         self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
         self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
 
-        self._spider_thread_num = setting.SPIDER_MAX_RETRIES
-        self._spider_max_retries = setting.SPIDER_THREAD_NUM
+        self._spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self._spider_thread_num = setting.SPIDER_THREAD_NUM
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
@@ -138,6 +138,7 @@ class Launcher(threading.Thread):
     def _remove_doing_seeds(self, seeds):
         for seed in seeds:
             self.__DOING__.pop(seed, None)
+        logger.info("remove %s seeds from __DOING__" % len(seeds))
 
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:

{cobweb-launcher-1.2.5 → cobweb-launcher-1.2.7}/cobweb/launchers/launcher_pro.py

@@ -36,31 +36,27 @@ class LauncherPro(Launcher):
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
-                self._db.setex(self._heartbeat_key, 3)
-            time.sleep(2)
+                self._db.setex(self._heartbeat_key, 5)
+            time.sleep(3)
 
     def _reset(self):
         """
         检查过期种子，重新添加到redis缓存中
         """
-        init = True
         while not self._pause.is_set():
             reset_wait_seconds = 30
-            start_reset_time = int(time.time())
             if self._db.lock(self._reset_lock_key, t=120):
-                if not self.heartbeat:
-                    self._heartbeat_start_event.set()
 
                 _min = -int(time.time()) + self._seed_reset_seconds \
-                    if self.heartbeat or not init else "-inf"
+                    if self.heartbeat else "-inf"
 
                 self._db.members(self._todo_key, 0, _min=_min, _max="(0")
                 self._db.delete(self._reset_lock_key)
 
-                ttl = 120 - int(time.time()) + start_reset_time
-                reset_wait_seconds = max(ttl, 1)
+                if not self.heartbeat:
+                    self._heartbeat_start_event.set()
+
             time.sleep(reset_wait_seconds)
-            init = False
 
     def _scheduler(self):
         """
@@ -111,7 +107,7 @@ class LauncherPro(Launcher):
                 refresh_time = int(time.time())
                 seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
                 self._db.zadd(self._todo_key, item=seeds, xx=True)
-            time.sleep(30)
+            time.sleep(15)
 
     def _delete(self):
         """

{cobweb-launcher-1.2.5 → cobweb-launcher-1.2.7}/cobweb/setting.py

@@ -26,9 +26,6 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
 OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
-# message
-MESSAGE = ""
-
 
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
@@ -43,9 +40,9 @@ BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次
 SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
 TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
-DONE_QUEUE_WAIT_SECONDS = 15   # done队列等待时间
+DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 300   # 种子重制时间
+SEED_RESET_SECONDS = 30   # 种子重制时间
 
 
 # Launcher 队列长度

{cobweb-launcher-1.2.5 → cobweb-launcher-1.2.7/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.5
+Version: 1.2.7
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.5 → cobweb-launcher-1.2.7}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.5",
+    version="1.2.7",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",