cobweb-launcher 0.1.18__tar.gz → 0.1.20__tar.gz

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.18
+Version: 0.1.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/cobweb/constant.py

@@ -20,8 +20,10 @@ class DealModel:
 class Setting:
     RESET_SCORE = None
     CHECK_LOCK_TIME = None
+    SCHEDULER_LOCK_TIME = None
     DEAL_MODEL = None
     LAUNCHER_MODEL = None
     SCHEDULER_WAIT_TIME = None
     SCHEDULER_BLOCK_TIME = None
     SPIDER_WAIT_TIME = None
+    SPIDER_SLEEP_TIME = None

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/cobweb/db/redis_db.py

@@ -119,7 +119,7 @@ class RedisDB:
     def get_seed(self, length: int = 200):
         cs = time.time()
 
-        if self._get_lock(key=self.update_lock):
+        if self._get_lock(key=self.update_lock, t=Setting.SCHEDULER_LOCK_TIME):
 
             update_item, result = {}, []
 

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/cobweb/equip/single/models.py

@@ -33,11 +33,11 @@ class Scheduler:
         while not stop.is_set():
 
             if not ready_seed_length():
-                time.sleep(5)
+                time.sleep(Setting.SCHEDULER_WAIT_TIME)
                 continue
 
             if self.queue.length >= self.length:
-                time.sleep(3)
+                time.sleep(Setting.SCHEDULER_BLOCK_TIME)
                 continue
 
             seeds = get_seed(self.length)
@@ -59,7 +59,7 @@ class Spider:
             seed = self.queue.pop()
 
             if not seed:
-                time.sleep(3)
+                time.sleep(Setting.SPIDER_WAIT_TIME)
                 continue
 
             elif seed._retry >= self.max_retries:
@@ -122,7 +122,7 @@ class Spider:
                 log.info(f"{str(seed)} -> {str(e)}")
             finally:
                 self.spider_in_progress.pop()
-                time.sleep(Setting.SPIDER_RUN_TIME)
+                time.sleep(Setting.SPIDER_SLEEP_TIME)
         log.info(f"close thread: spider")
 
 

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/cobweb/task.py

@@ -6,11 +6,13 @@ from .utils import parse_info, struct_start_seeds
 def init_task_env():
     Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
     Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+    Setting.SCHEDULER_LOCK_TIME = int(os.getenv("SCHEDULER_LOCK_TIME", 15))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
     Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))
     Setting.SCHEDULER_BLOCK_TIME = float(os.getenv("SCHEDULER_BLOCK_TIME", 3))
     Setting.SPIDER_WAIT_TIME = float(os.getenv("SPIDER_WAIT_TIME", 3))
+    Setting.SPIDER_SLEEP_TIME = float(os.getenv("SPIDER_SLEEP_TIME", 0.5))
 
 
 class Task:

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.18
+Version: 0.1.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.18 → cobweb-launcher-0.1.20}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.1.18",
+    version="0.1.20",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",