cobweb-launcher 0.1.16__py3-none-any.whl → 0.1.17__py3-none-any.whl

cobweb/__init__.py

@@ -3,6 +3,7 @@ from .task import Task
 from .log import log
 from .db.redis_db import RedisDB
 from .db.oss_db import OssDB
+from .constant import Setting
 
 from .equip.distributed.launcher import launcher
 from .equip.single.launcher import launcher as single_launcher

cobweb/constant.py

@@ -22,4 +22,6 @@ class Setting:
     CHECK_LOCK_TIME = None
     DEAL_MODEL = None
     LAUNCHER_MODEL = None
-    SPIDER_RUN_TIME = None
+    SCHEDULER_WAIT_TIME = None
+    SCHEDULER_BLOCK_TIME = None
+    SPIDER_WAIT_TIME = None

cobweb/single/models.py

@@ -1,5 +1,5 @@
 import time
-from cobweb import log, Queue, Seed
+from cobweb import log, Queue, Seed, Setting
 from cobweb.utils import issubclass_cobweb_inf
 # from pympler import asizeof
 
@@ -32,11 +32,11 @@ class Scheduler:
         while not stop.is_set():
 
             if not ready_seed_length():
-                time.sleep(5)
+                time.sleep(Setting.SCHEDULER_WAIT_TIME)
                 continue
 
             if self.queue.length >= self.length:
-                time.sleep(3)
+                time.sleep(Setting.SCHEDULER_BLOCK_TIME)
                 continue
 
             seeds = get_seed(self.length)
@@ -55,7 +55,7 @@ class Spider:
         while not stop.is_set():
             seed = self.queue.pop()
             if not seed:
-                time.sleep(3)
+                time.sleep(Setting.SPIDER_WAIT_TIME)
                 continue
             elif seed._retry >= self.max_retries:
                 del_seed(seed, spider_status=False)

cobweb/task.py

@@ -9,6 +9,9 @@ def init_task_env():
     Setting.SPIDER_RUN_TIME = float(os.getenv("SPIDER_RUN_TIME", 0.2))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
+    Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))
+    Setting.SCHEDULER_BLOCK_TIME = float(os.getenv("SCHEDULER_BLOCK_TIME", 3))
+    Setting.SPIDER_RUN_TIME = float(os.getenv("SPIDER_RUN_TIME", 3))
 
 
 class Task:

{cobweb_launcher-0.1.16.dist-info → cobweb_launcher-0.1.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.16
+Version: 0.1.17
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.1.16.dist-info → cobweb_launcher-0.1.17.dist-info}/RECORD

@@ -1,11 +1,11 @@
-cobweb/__init__.py,sha256=rd5vkgxe49DzVH4oyvwl-pWQeJrqF1K6XT1NZfvMmTo,256
+cobweb/__init__.py,sha256=zVims_awYxKNqULuKSjFXdZAaDJaqzFBJJo_SqeakMs,286
 cobweb/bbb.py,sha256=zKeCeBVFQfhEv6M8CCktUTM7tXDZmAu6ZN0-ET44pUY,5707
-cobweb/constant.py,sha256=t0_2xY_VU6T18DnD8tL9LDQJB6ibE9UWx66H5yz_934,489
+cobweb/constant.py,sha256=ytkGczukfFU9bx4ricEQy7eY3Odwc4Xmrps9PwuGiuo,553
 cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
 cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
 cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
 cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
-cobweb/task.py,sha256=okE6rBFm9VnYOAXShuTXJ0uvc-_ITsuuxhyxNKbymmU,1775
+cobweb/task.py,sha256=wqJKyMYg_ftCB29mTTlEue5e2zq-5tkC4mFFqraJwvA,2000
 cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
 cobweb/db/__init__.py,sha256=jC-uOThYLtiDUG6cTJRkDITgOzR4nIOeaZQeZhuk-v0,139
 cobweb/db/oss_db.py,sha256=59HwMMfoGUr6l_BI4p84YHYoQwEn22cCM_1GZK34Uo4,4244
@@ -33,10 +33,10 @@ cobweb/equip/single/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4H
 cobweb/equip/single/models.py,sha256=pWALuEat5v9cNAvJ_PaAlDcIHUWuu38EQl0KFcpSJXA,5344
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
-cobweb/single/models.py,sha256=UXcxr_Quok91k82plaqbj4deB-UBCWo14WCo6SS5L_o,4247
+cobweb/single/models.py,sha256=wIEV35666lxdzqjDqBHPjjh-r6zD0x24rtQYz7d4Oxw,4332
 cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
-cobweb_launcher-0.1.16.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.1.16.dist-info/METADATA,sha256=5LhIX1XNOr4pfnbotXN-lNWdwLUdBeb5Q3O157KORns,1220
-cobweb_launcher-0.1.16.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cobweb_launcher-0.1.16.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.1.16.dist-info/RECORD,,
+cobweb_launcher-0.1.17.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.1.17.dist-info/METADATA,sha256=Q5ZOsZqe_ETrAg5opFlThYQHk3kZUcYqPEB4BZeD9JY,1220
+cobweb_launcher-0.1.17.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cobweb_launcher-0.1.17.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.1.17.dist-info/RECORD,,