cobweb-launcher 1.2.5__py3-none-any.whl → 1.2.6__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -86,7 +86,7 @@ class Crawler(threading.Thread):
                 time.sleep(1)
                 continue
 
-            elif seed.params.retry >= self.max_retries:
+            elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
                 self._done.push(seed)
                 continue

cobweb/launchers/launcher.py

@@ -79,8 +79,8 @@ class Launcher(threading.Thread):
         self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
         self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
 
-        self._spider_thread_num = setting.SPIDER_MAX_RETRIES
-        self._spider_max_retries = setting.SPIDER_THREAD_NUM
+        self._spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self._spider_thread_num = setting.SPIDER_THREAD_NUM
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL

cobweb/launchers/launcher_pro.py

@@ -36,31 +36,27 @@ class LauncherPro(Launcher):
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
-                self._db.setex(self._heartbeat_key, 3)
-            time.sleep(2)
+                self._db.setex(self._heartbeat_key, 5)
+            time.sleep(3)
 
     def _reset(self):
         """
         检查过期种子，重新添加到redis缓存中
         """
-        init = True
         while not self._pause.is_set():
             reset_wait_seconds = 30
-            start_reset_time = int(time.time())
             if self._db.lock(self._reset_lock_key, t=120):
-                if not self.heartbeat:
-                    self._heartbeat_start_event.set()
 
                 _min = -int(time.time()) + self._seed_reset_seconds \
-                    if self.heartbeat or not init else "-inf"
+                    if self.heartbeat else "-inf"
 
                 self._db.members(self._todo_key, 0, _min=_min, _max="(0")
                 self._db.delete(self._reset_lock_key)
 
-                ttl = 120 - int(time.time()) + start_reset_time
-                reset_wait_seconds = max(ttl, 1)
+                if not self.heartbeat:
+                    self._heartbeat_start_event.set()
+
             time.sleep(reset_wait_seconds)
-            init = False
 
     def _scheduler(self):
         """
@@ -111,7 +107,7 @@ class LauncherPro(Launcher):
                 refresh_time = int(time.time())
                 seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
                 self._db.zadd(self._todo_key, item=seeds, xx=True)
-            time.sleep(30)
+            time.sleep(15)
 
     def _delete(self):
         """

cobweb/setting.py

@@ -45,7 +45,7 @@ TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
 DONE_QUEUE_WAIT_SECONDS = 15   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 300   # 种子重制时间
+SEED_RESET_SECONDS = 30   # 种子重制时间
 
 
 # Launcher 队列长度

{cobweb_launcher-1.2.5.dist-info → cobweb_launcher-1.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.5
+Version: 1.2.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.5.dist-info → cobweb_launcher-1.2.6.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=_t3LMSpxUNR4dVD9Tox22W9omHPvjeWABFzpgkEOoH0,2016
+cobweb/setting.py,sha256=bKM2J-yjGHeQn1Yf-yZ3cay_FTCPaokgOYSC6GPWwM4,2015
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -11,16 +11,16 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=_4bU4rlg-oQTztWibtAEKY4nzFxpZ-FBu6KuwhEkewE,6003
+cobweb/crawlers/crawler.py,sha256=MJ2rFTrVjQ4Q9TmX0dpbOvT4tdokQwh7dKeFY_W7pPE,6002
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=Mepg-hv9YL5_VnY_Mwqxzd8ZRGyK8gxsl_5B_Ibz0Uc,5466
+cobweb/launchers/launcher.py,sha256=0YhJtmN048MgTZ7SAUWvaKJEPXrNx0p4dkoTopsyjzc,5466
 cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
-cobweb/launchers/launcher_pro.py,sha256=g0aqlUSmgU4o7F9bQ_oYrdlqn8w71wcIS93t1NvzxBg,8324
+cobweb/launchers/launcher_pro.py,sha256=qCmobbi1jhI09cVRjaYGxyTHwBSvARiHxF1p37b5iaE,8107
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
@@ -30,8 +30,8 @@ cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXt
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.5.dist-info/METADATA,sha256=-1P8DAthZeUuTBjWMQDUG0GmVIctOEGM4dc3Kh175-8,6489
-cobweb_launcher-1.2.5.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.5.dist-info/RECORD,,
+cobweb_launcher-1.2.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.6.dist-info/METADATA,sha256=q_ihYp0LYgZRMdvzJ2gmTYG0HgeoOAnRjOEj5FAUp34,6489
+cobweb_launcher-1.2.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.6.dist-info/RECORD,,